From bc2f4e3a789dfab29b5fe65416dd54909fe2853e Mon Sep 17 00:00:00 2001 From: Ryan Newton Date: Tue, 6 Dec 2022 12:19:02 -0800 Subject: [PATCH] add --fuzz-seed and --fuzz-futexes Summary: This adds the first non-thread-order, non-RNG source of fuzzing, as described in this issue: https://github.com/facebookexperimental/hermit/issues/34 Before this diff, hermit is choosing an arbitrary (but deterministic) set of waiting threads to wake on each FUTEX_WAKE. After this diff, that selection is randomized, using deterministic PRNG from a new seed. Reviewed By: jasonwhite Differential Revision: D41721535 fbshipit-source-id: 8c340df049ffa0792cfaaead20d7441d6145c2b7 --- detcore-model/src/config.rs | 16 ++++++++++++ detcore/src/scheduler.rs | 42 +++++++++++++++++++++++++++--- detcore/tests/testutils/src/lib.rs | 6 +++++ hermit-cli/src/bin/hermit/run.rs | 6 +++++ hermit-cli/src/metadata.rs | 2 ++ tests/rust/futex_wake_some.rs | 3 ++- 6 files changed, 71 insertions(+), 4 deletions(-) diff --git a/detcore-model/src/config.rs b/detcore-model/src/config.rs index 998a187..51a2d60 100644 --- a/detcore-model/src/config.rs +++ b/detcore-model/src/config.rs @@ -65,6 +65,12 @@ pub struct Config { #[clap(long, value_name = "uint64")] pub rng_seed: Option, + /// Seeds the PRNG which drives syscall response fuzzing (i.e. chaotically exercising syscall + /// nondeterminism). Like other seeds, this is initialized from the `--seed` if not + /// specifically provided. + #[clap(long, value_name = "uint64")] + pub fuzz_seed: Option, + /// Logical clock multiplier. Values above one make time appear to go faster within the sandbox. #[clap(long, value_name = "float")] pub clock_multiplier: Option, @@ -93,6 +99,10 @@ pub struct Config { #[clap(long)] pub chaos: bool, + /// Uses the `--fuzz-seed` to generate randomness and fuzz nondeterminism in the futex semantics. + #[clap(long)] + pub fuzz_futexes: bool, + /// Record the timing of preemption events for future replay or experimentation. /// This is only useful in chaos modes. #[clap(long)] @@ -566,6 +576,12 @@ impl Config { self.rng_seed.unwrap_or(self.seed) } + /// Returns the fuzz_seed, as specified by the user or defaulting to the primary seed if + /// unspecified. + pub fn fuzz_seed(&self) -> u64 { + self.fuzz_seed.unwrap_or(self.seed) + } + /// Returns effective "sched-seed" parameter taking in account "seed" /// parameter if former isn't specified pub fn sched_seed(&self) -> u64 { diff --git a/detcore/src/scheduler.rs b/detcore/src/scheduler.rs index 90182a7..92f2c5b 100644 --- a/detcore/src/scheduler.rs +++ b/detcore/src/scheduler.rs @@ -28,6 +28,9 @@ use detcore_model::collections::ReplayCursor; use nix::sys::signal; use nix::sys::signal::Signal; use nix::unistd::Pid; +use rand::seq::SliceRandom; +use rand::SeedableRng; +use rand_pcg::Pcg64Mcg; use reverie::syscalls::Syscall; use reverie::syscalls::SyscallInfo; pub use runqueue::entropy_to_priority; @@ -202,7 +205,7 @@ fn assert_continue_request(req: &Resources) { } /// The state for the deterministic scheduler. -#[derive(Debug, Default)] +#[derive(Debug)] pub struct Scheduler { /// Monotonically count upwards. pub turn: u64, @@ -289,6 +292,9 @@ pub struct Scheduler { /// the (original) replay_cursor trace. pub stacktrace_events: Option, + /// PRNG to drive any fuzzing of OS semantics (other than scheduling). + fuzz_prng: Pcg64Mcg, + /// A cached copy of the same (immutable) field in Config. stop_after_turn: Option, /// A cached copy of the same (immutable) field in Config. @@ -299,6 +305,8 @@ pub struct Scheduler { die_on_desync: bool, /// A cached copy of the same (immutable) field in Config. replay_exhausted_panic: bool, + /// A cached copy of the same (immutable) field in Config. + fuzz_futexes: bool, } type StacktraceEventsIter = Peekable)>>; @@ -860,6 +868,8 @@ impl Scheduler { thread_tree: Default::default(), priorities: Default::default(), timeslices: Default::default(), + fuzz_futexes: cfg.fuzz_futexes, + fuzz_prng: Pcg64Mcg::seed_from_u64(cfg.fuzz_seed()), } } @@ -1180,8 +1190,33 @@ impl Scheduler { // is ready to run in normal order. } + fn choose_futex_wakees( + &mut self, + vec: &mut Vec<(DetPid, Ivar)>, + num_woken: usize, + ) -> Vec<(DetPid, Ivar)> { + if self.fuzz_futexes { + let rng = &mut self.fuzz_prng; + debug!( + "[fuzz-futexes] selecting {} tids, pre shuffle: {:?}", + num_woken, + vec.iter().map(|x| x.0).collect::>() + ); + + // No need to actually use the results here since vec was mutated: + let (_extracted, _remain) = &vec[..].partial_shuffle(rng, num_woken); + + info!( + "[fuzz-futexes] selecting {} tids, post shuffle: {:?}", + num_woken, + vec.iter().map(|x| x.0).collect::>() + ); + } + // just take the first N, in whatever deterministic order they are in: + vec.split_off(vec.len() - num_woken) + } + /// Reschedule all threads blocked on a particular futex. - /// TODO: support rescheduling exactly K threads. pub fn wake_futex_waiters( &mut self, _waker_dettid: DetTid, @@ -1210,7 +1245,8 @@ impl Scheduler { vec.len(), ); let num_woken: usize = std::cmp::min(vec.len(), max_to_wake.try_into().unwrap()); - let to_wake = vec.split_off(vec.len() - num_woken); + let to_wake = self.choose_futex_wakees(&mut vec, num_woken); + assert_eq!(to_wake.len(), num_woken); for waiter in to_wake { self.wake_futex_waiter(waiter); diff --git a/detcore/tests/testutils/src/lib.rs b/detcore/tests/testutils/src/lib.rs index 7780cc8..06f1479 100644 --- a/detcore/tests/testutils/src/lib.rs +++ b/detcore/tests/testutils/src/lib.rs @@ -66,6 +66,7 @@ lazy_static! { seed: DEFAULT_CFG.seed, rng_seed: None, sched_seed: None, + fuzz_seed: None, gdbserver: false, gdbserver_port: 1234, preemption_timeout: NonZeroU64::new(5000000), @@ -94,6 +95,7 @@ lazy_static! { sysinfo_uptime_offset: 60, memory: 1024 * 1024 * 1024, //1 GiB interrupt_at: vec![], + fuzz_futexes: false, }; /// Standardized test config: common options on. @@ -115,6 +117,7 @@ lazy_static! { seed: DEFAULT_CFG.seed, rng_seed: None, sched_seed: None, + fuzz_seed: None, gdbserver: false, gdbserver_port: 1234, preemption_timeout: NonZeroU64::new(5000000), @@ -143,6 +146,7 @@ lazy_static! { sysinfo_uptime_offset: 60, memory: 1024 * 1024 * 1024, //1 GiB interrupt_at: vec![], + fuzz_futexes: false, }; /// Standardized test config: all options on. @@ -164,6 +168,7 @@ lazy_static! { seed: DEFAULT_CFG.seed, rng_seed: None, sched_seed: None, + fuzz_seed: None, gdbserver: false, gdbserver_port: 1234, preemption_timeout: NonZeroU64::new(5000000), @@ -192,6 +197,7 @@ lazy_static! { sysinfo_uptime_offset: 60, memory: 1024 * 1024 * 1024, //1 GiB interrupt_at: vec![], + fuzz_futexes: false, }; } diff --git a/hermit-cli/src/bin/hermit/run.rs b/hermit-cli/src/bin/hermit/run.rs index 4f3d254..f440b27 100644 --- a/hermit-cli/src/bin/hermit/run.rs +++ b/hermit-cli/src/bin/hermit/run.rs @@ -465,7 +465,13 @@ impl fmt::Display for RunOpts { if let Some(rng_seed) = dop.rng_seed { write!(f, " --rng-seed={}", rng_seed)?; } + if let Some(fuzz_seed) = dop.fuzz_seed { + write!(f, " --fuzz-seed={}", fuzz_seed)?; + } + if dop.fuzz_futexes { + write!(f, " --fuzz-futexes")?; + } if let Some(m) = dop.clock_multiplier { write!(f, " --clock-multiplier={}", m)?; } diff --git a/hermit-cli/src/metadata.rs b/hermit-cli/src/metadata.rs index 3cac7a2..c303f59 100644 --- a/hermit-cli/src/metadata.rs +++ b/hermit-cli/src/metadata.rs @@ -193,6 +193,8 @@ pub fn record_or_replay_config(data: &Path) -> detcore::Config { sysinfo_uptime_offset: 120, memory: 1024 * 1024 * 1024, interrupt_at: vec![], + fuzz_futexes: false, + fuzz_seed: None, }; if config.preemption_timeout.is_some() && !reverie_ptrace::is_perf_supported() { tracing::warn!( diff --git a/tests/rust/futex_wake_some.rs b/tests/rust/futex_wake_some.rs index 6c08d3f..6a7f5ea 100644 --- a/tests/rust/futex_wake_some.rs +++ b/tests/rust/futex_wake_some.rs @@ -16,7 +16,7 @@ use nix::errno::errno; use nix::errno::Errno; const TOTAL: i64 = 4; -const TO_WAKE: i64 = 3; +const TO_WAKE: i64 = 2; fn main() { let layout = std::alloc::Layout::from_size_align(4, 4).unwrap(); // u32 @@ -45,6 +45,7 @@ fn main() { }; if val == 0 { children_post.fetch_add(1, Ordering::SeqCst); + eprintln!("tid {} woken", ix); } else { println!( "UNLIKELY: child thread {} issued wait after wake already happened, errno {}",