fuzz: introduce canaries

firedancer-io · Dec 1, 2023 · 5fdba9e · 5fdba9e
1 parent c454a06
commit 5fdba9e
Show file tree

Hide file tree

Showing 8 changed files with 159 additions and 4 deletions.
diff --git a/.github/workflows/on_daily.yml b/.github/workflows/on_daily.yml
@@ -50,3 +50,6 @@ jobs:
           <meta http-equiv="cache-control" content="no-cache" /></head>
           <body>Redirecting to latest coverage... (${{ github.sha }})</body></html>' > ./build/index.html
           gcloud storage cp -r ./build/index.html ${{ vars.COVERAGE_BUCKET }}/
+
+      - name: Check fuzz canaries
+        run: ./contrib/find_uncovered_fuzz_canaries.py $(find ./build/ -name 'fuzz_*\.lcov' -type f)
diff --git a/config/base.mk b/config/base.mk
@@ -31,6 +31,7 @@ RUST_PROFILE=debug
 
 # lcov
 GENHTML=genhtml
+# newer versions of genhtml will require '-ignore-errors unmapped'
 
 # FD_HAS_MAIN: Target supports linking objects with main function.
 # If set to 0, programs and unit tests will not be built. This is

diff --git a/config/with-fuzz.mk b/config/with-fuzz.mk
@@ -1,5 +1,8 @@
 FD_HAS_MAIN:=0
-CPPFLAGS+=-fno-omit-frame-pointer
+FD_HAS_FUZZ:=1
 
+CPPFLAGS+=-DFD_HAS_FUZZ=1
+CPPFLAGS+=-fno-omit-frame-pointer
 CPPFLAGS+=-fsanitize=fuzzer-no-link
+
 LDFLAGS+=-fsanitize=fuzzer
diff --git a/contrib/find_uncovered_fuzz_canaries.py b/contrib/find_uncovered_fuzz_canaries.py
@@ -0,0 +1,114 @@
+#!/usr/bin/env python3
+
+# run this script from the top of the repository
+
+from collections import defaultdict
+import functools
+import os
+import subprocess
+import sys
+
+
+canary_canary_path = 'src/util/sanitize/test_fuzz_canary_canary.c'
+
+def main(lcov_files):
+    canaries = find_canaries()
+
+    # turn canaries into map of (file_name -> {linum, ...})
+    def add(a, c):
+        a[c.file].add(c.linum)
+        return a
+    files_to_lines = functools.reduce(add, canaries, defaultdict(set))
+
+    print(f"canaries found in source ({len(canaries)}):")
+    for entry in files_to_lines.items():
+        print(f"\t{entry}")
+
+    for lcov_path in lcov_files:
+        files_to_lines = eliminate_canaries_with_lcov(lcov_path, files_to_lines)
+
+    live_canaries = files_to_lines
+    print("live canaries", live_canaries)
+
+    # check for canary canary - if absent, this tool has failed.
+    if len(live_canaries[canary_canary_path]) != 1:
+        print(f"the canary in {canary_canary_path} hasn't been found as uncovered - the tool is faulty", file=sys.stderr)
+        os.exit(1)
+
+    # remove the canary canary from the the findings
+    del live_canaries[canary_canary_path]
+
+    live_canaries = list(filter(lambda item: len(item[1]) != 0, live_canaries.items()))
+
+    if not live_canaries:
+        print("no uncovered canaries")
+
+    else:
+        for (canary_file, linums) in live_canaries:
+            for linum in linums:
+                print(f"::warning file={canary_file},line={linum}::\"Canary not yet covered by fuzzing\"")
+
+
+class Canary:
+    def __init__(self, file, linum):
+        self.file = file
+        self.linum = linum
+
+def find_canaries():
+    # define the command to be executed
+    cmd = ["grep", "-Hn", "-r", "--exclude=fd_fuzz.h", "FD_FUZZ_MUST_BE_COVERED", "src/"]
+    try:
+        # execute the command
+        result = subprocess.run(cmd, check=True, stdout=subprocess.PIPE, text=True)
+
+        # split the output into an array, one element per line
+        output_lines = result.stdout.strip().split('\n')
+
+        output_tokens = []
+        for line in output_lines:
+            raw_canary = line.split(":", 2)
+            output_tokens.append(Canary(raw_canary[0], raw_canary[1]))
+
+        return output_tokens
+
+    except subprocess.CalledProcessError as e:
+        # Handle errors such as directory not found or grep command failure
+        return ["Error: " + str(e)]
+
+# eliminate_canaries_with_lcov reads an lcov file at the specified path
+# and returns all of the canaries that were not covered
+def eliminate_canaries_with_lcov(path_to_lcov, canaries):
+    print(f"processing file: {path_to_lcov}")
+    cwd = os.getcwd()
+    with open(path_to_lcov) as lcov:
+
+        # loop state
+        node_of_interest = None
+        source_path = None
+        while True: # go over every line in the lcov
+            line = lcov.readline()
+            if line == "":
+                break
+
+            if line.startswith("SF:"): # new source file context: track node if applicable
+                source_path = line[len(f"SF:{cwd}/"):].strip() # make path relative
+                node_of_interest = canaries.get(source_path, None)
+
+            elif line.startswith("end_of_context"):
+                source_path = None
+                node_of_interest = None
+
+            elif line.startswith("DA:"):
+                if node_of_interest is not None:
+                    linum, hits = line[3:].split(",", 1)
+
+                    if hits.strip() == "0": # lcov line has no hits thus is uncovered
+                        continue
+
+                    node_of_interest.discard(linum)
+        return canaries
+
+
+if __name__ == '__main__':
+    print("using lcov files:", sys.argv[1:])
+    main(sys.argv[1:])
diff --git a/src/ballet/sbpf/fuzz_sbpf_loader.c b/src/ballet/sbpf/fuzz_sbpf_loader.c
@@ -2,6 +2,7 @@
 #error "This target requires FD_HAS_HOSTED"
 #endif
 
+#include "../../util/sanitize/fd_fuzz.h"
 #include "fd_sbpf_loader.h"
 #include "fd_sbpf_maps.c"
 
@@ -35,7 +36,7 @@ LLVMFuzzerTestOneInput( uchar const * data,
 
   fd_sbpf_elf_info_t info;
   if( FD_UNLIKELY( !fd_sbpf_elf_peek( &info, data, size ) ) )
-    return 0;
+    return -1;
 
   /* Allocate objects */
 
@@ -53,12 +54,17 @@ LLVMFuzzerTestOneInput( uchar const * data,
 
   /* Load program */
   int res = fd_sbpf_program_load( prog, data, size, syscalls );
-  FD_COMPILER_FORGET( res );
+
+  /* Should be able to load at least one program and not load at least one program */
+  if ( FD_UNLIKELY( !res ) ) {
+    FD_FUZZ_MUST_BE_COVERED;
+  } else {
+    FD_FUZZ_MUST_BE_COVERED;
+  }
 
   /* Clean up */
   free( rodata );
   free( fd_sbpf_syscalls_delete( syscalls ) );
   free( fd_sbpf_program_delete( prog ) );
   return 0;
 }
-
diff --git a/src/util/fd_util_base.h b/src/util/fd_util_base.h
@@ -179,6 +179,12 @@
 #define FD_HAS_GFNI 0
 #endif
 
+/* FD_HAS_FUZZ indicates that the build target is a fuzz target. */
+
+#ifndef FD_HAS_FUZZ
+#define FD_HAS_FUZZ 0
+#endif
+
 /* FD_HAS_ASAN indicates that the build target is using ASAN. */
 
 #ifndef FD_HAS_ASAN

diff --git a/src/util/sanitize/fd_fuzz.h b/src/util/sanitize/fd_fuzz.h
@@ -0,0 +1,11 @@
+#ifndef HEADER_fd_src_util_fuzz_fd_fuzz_h
+#define HEADER_fd_src_util_fuzz_fd_fuzz_h
+
+#if FD_HAS_FUZZ
+#define FD_FUZZ_MUST_BE_COVERED ((void) 0)
+#else
+#define FD_FUZZ_MUST_BE_COVERED
+#endif
+
+
+#endif
diff --git a/src/util/sanitize/test_fuzz_canary_canary.c b/src/util/sanitize/test_fuzz_canary_canary.c
@@ -0,0 +1,11 @@
+/* This files contains a canary that is expected to be found by the canary finder.
+   If the script fails to find this canary, it will consider this a failure. 
+   
+   This is not a unit test but a canary. */
+
+#include "fd_fuzz.h"
+
+static void
+do_not_call_me( void ) {
+    FD_FUZZ_MUST_BE_COVERED;
+}