From 17744f53a372b105d034450fa4cd8c5e98493b13 Mon Sep 17 00:00:00 2001 From: Michael Rodler Date: Thu, 4 Jan 2024 11:51:58 +0100 Subject: [PATCH 1/2] More flexibility when creating fuzzers at the cost of Fuzzers requiring `Clone + Send`. * Added `Fuzzer::load_seed_input` to allow additional or different handling of seed inputs, e.g., parsing seed files that are then stored in a different format internally and in the corpus (e.g., parsing source code into an AST and then doing AST mutations). * A fuzzer is now constructed with `FUZZER::new` once and then cloned for every core. As a consequence fuzzers are now `Clone + Send`. This allows for several patterns: * Custom shared state across all cores, e.g., for custom metadata. * Performing costly initialization only once (e.g., parsing a system call definition file). --- examples/01_getpid/src/fuzzer.rs | 2 +- examples/02_libtiff/src/fuzzer.rs | 2 +- .../03_ffmpeg_custom_mutator/src/fuzzer.rs | 2 +- examples/04_syscall_fuzzer/src/fuzzer.rs | 2 +- examples/05_redqueen/src/fuzzer.rs | 2 +- examples/06_custom_feedback/src/fuzzer.rs | 2 +- examples/07_libfuzzer/src/fuzzer.rs | 2 +- src/commands/fuzz.rs | 94 ++++++++++++------- src/fuzzer.rs | 14 ++- 9 files changed, 82 insertions(+), 40 deletions(-) diff --git a/examples/01_getpid/src/fuzzer.rs b/examples/01_getpid/src/fuzzer.rs index a7513ce..7f18da5 100644 --- a/examples/01_getpid/src/fuzzer.rs +++ b/examples/01_getpid/src/fuzzer.rs @@ -16,7 +16,7 @@ use crate::constants; const CR3: Cr3 = Cr3(constants::CR3); -#[derive(Default)] +#[derive(Default, Clone)] pub struct Example1Fuzzer { // Fuzzer specific data could go in here } diff --git a/examples/02_libtiff/src/fuzzer.rs b/examples/02_libtiff/src/fuzzer.rs index 15a368a..44d011d 100644 --- a/examples/02_libtiff/src/fuzzer.rs +++ b/examples/02_libtiff/src/fuzzer.rs @@ -16,7 +16,7 @@ use crate::constants; const CR3: Cr3 = Cr3(constants::CR3); -#[derive(Default)] +#[derive(Default, Clone)] pub struct Example02Fuzzer { // Fuzzer specific data could go in here } diff --git a/examples/03_ffmpeg_custom_mutator/src/fuzzer.rs b/examples/03_ffmpeg_custom_mutator/src/fuzzer.rs index 284cb6d..b26f546 100644 --- a/examples/03_ffmpeg_custom_mutator/src/fuzzer.rs +++ b/examples/03_ffmpeg_custom_mutator/src/fuzzer.rs @@ -15,7 +15,7 @@ use snapchange::linux::{read_args, ReadArgs}; use snapchange::rng::Rng; use snapchange::Execution; -#[derive(Default)] +#[derive(Default, Clone)] pub struct Example03Fuzzer { file_offset: usize, } diff --git a/examples/04_syscall_fuzzer/src/fuzzer.rs b/examples/04_syscall_fuzzer/src/fuzzer.rs index a3dd9e3..0639477 100644 --- a/examples/04_syscall_fuzzer/src/fuzzer.rs +++ b/examples/04_syscall_fuzzer/src/fuzzer.rs @@ -82,7 +82,7 @@ const SHELLCODE: u64 = constants::SHELLCODE; const SCRATCH: u64 = constants::SCRATCH; const CR3: Cr3 = Cr3(constants::CR3); -#[derive(Default)] +#[derive(Default, Clone)] pub struct Example04Fuzzer { /// Offset to the next address to write shellcode shellcode_offset: u64, diff --git a/examples/05_redqueen/src/fuzzer.rs b/examples/05_redqueen/src/fuzzer.rs index 48435a1..5720c44 100644 --- a/examples/05_redqueen/src/fuzzer.rs +++ b/examples/05_redqueen/src/fuzzer.rs @@ -12,7 +12,7 @@ const CR3: Cr3 = Cr3(constants::CR3); // src/fuzzer.rs -#[derive(Default)] +#[derive(Default, Clone)] pub struct Example05Fuzzer; impl Fuzzer for Example05Fuzzer { diff --git a/examples/06_custom_feedback/src/fuzzer.rs b/examples/06_custom_feedback/src/fuzzer.rs index f1092b4..f1f26e8 100644 --- a/examples/06_custom_feedback/src/fuzzer.rs +++ b/examples/06_custom_feedback/src/fuzzer.rs @@ -80,7 +80,7 @@ impl std::default::Default for WasdArray { } } -#[derive(Default)] +#[derive(Default, Clone)] pub struct MazeFuzzer { /// this is used for input scheduling - assigning weights to each input. The latest corpus entry /// has the biggest weight, while the first one has the smallest. Essentially this one is a diff --git a/examples/07_libfuzzer/src/fuzzer.rs b/examples/07_libfuzzer/src/fuzzer.rs index 2d4701d..3ca34d2 100644 --- a/examples/07_libfuzzer/src/fuzzer.rs +++ b/examples/07_libfuzzer/src/fuzzer.rs @@ -12,7 +12,7 @@ use snapchange::fuzzvm::FuzzVm; use crate::constants; -#[derive(Default)] +#[derive(Default, Clone)] pub struct Example7Fuzzer {} impl Fuzzer for Example7Fuzzer { diff --git a/src/commands/fuzz.rs b/src/commands/fuzz.rs index 593af3b..cfa8080 100644 --- a/src/commands/fuzz.rs +++ b/src/commands/fuzz.rs @@ -107,41 +107,37 @@ pub(crate) fn run( log::warn!("Starting all {} worker threads", cores); + let mut fuzzer = FUZZER::new(); + // Read the input corpus from the given input directory let mut input_corpus: Vec>> = Vec::new(); // Use the user given input directory or default to /input let input_dir = if let Some(input_dir) = &args.input_dir { input_dir.clone() - } else if project_state.path.join("current_corpus").exists() { - // If no input dir was given, and the current corpus exists, use the old corpus - project_state.path.join("current_corpus") - } else if project_state.path.join("input").exists() { - // If no given input or current corpus, use "input" directory - project_state.path.join("input") } else { - // Default to the standard current_corpus directory - project_state.path.join("current_corpus") + project_state.path.join("input") }; // Get the corpus directory let mut corpus_dir = project_state.path.clone(); corpus_dir.push("current_corpus"); if !corpus_dir.exists() { - std::fs::create_dir(&corpus_dir).context("Failed to create crash dir")?; + std::fs::create_dir(&corpus_dir).context("Failed to create corpus dir")?; } - - let num_files = input_dir.read_dir()?.count(); + let num_corpus = corpus_dir.read_dir()?.count(); // Give some statistics on reading the initial corpus let mut start = std::time::Instant::now(); let mut count = 0_u32; + let mut total_loaded = 0_u32; if input_dir.exists() { + let num_files = input_dir.read_dir()?.count(); for (i, file) in input_dir.read_dir()?.enumerate() { if start.elapsed() >= std::time::Duration::from_millis(1000) { let left = num_files - i; println!( - "{i:9} / {num_files:9} | Reading corpus {:8.2} files/sec | {:6.2} seconds left", + "{i:9} / {num_files:9} | Reading seeds {:8.2} files/sec | {:6.2} seconds left", count as f64 / start.elapsed().as_secs_f64(), left as f64 / (count as f64 / start.elapsed().as_secs_f64()), ); @@ -158,29 +154,58 @@ pub(crate) fn run( log::debug!("Ignoring directory found in input dir: {:?}", filepath); continue; } + + // Add the input to the input corpus + let input = fuzzer.load_seed_input(&filepath, &project_state.path)?; + + let input = Arc::new(input); + input_corpus.push(input); + + total_loaded += 1; + } + } else { + log::info!("no seed input directory found: {}", input_dir.display()); + } + start = std::time::Instant::now(); + count = 0_u32; + if num_corpus > 0 { + for (i, file) in corpus_dir.read_dir()?.enumerate() { + if start.elapsed() >= std::time::Duration::from_millis(1000) { + let left = num_corpus - i; + println!( + "{i:9} / {num_corpus:9} | Reading current corpus {:8.2} files/sec | {:6.2} seconds left", + count as f64 / start.elapsed().as_secs_f64(), + left as f64 / (count as f64 / start.elapsed().as_secs_f64()), + ); + start = std::time::Instant::now(); + count = 0; + } + + count += 1; + + let filepath = file?.path(); + + // Ignore directories if they exist + if filepath.is_dir() { + log::debug!("Ignoring directory found in corpus dir: {:?}", filepath); + continue; + } + // Add the input to the input corpus - let input = FUZZER::Input::from_bytes(&std::fs::read(filepath)?)?; - let metadata_path = project_state - .path - .join("metadata") - .join(crate::utils::hexdigest(&input)); - - let input = if let Ok(data) = std::fs::read_to_string(metadata_path) { - let metadata = serde_json::from_str(&data)?; - InputWithMetadata { - input, - metadata: RwLock::new(metadata), - } - } else { - InputWithMetadata::from_input(input) - }; + let input = InputWithMetadata::::from_path(&filepath, &project_state.path)?; let input = Arc::new(input); input_corpus.push(input); + + total_loaded += 1; } } else { - log::warn!("No input directory found: {input_dir:?}, starting with an empty corpus!"); + log::info!("starting with an empty corpus"); + } + + if total_loaded == 0 { + log::warn!("No inputs loaded - starting with an empty corpus!"); } // Initialize the dictionary @@ -357,6 +382,9 @@ pub(crate) fn run( start.elapsed() ); + + let fuzzer = fuzzer; + const BETWEEN_WAIT_FOR_MILLIES: u64 = 100; // Create a thread for each active CPU core. @@ -445,10 +473,13 @@ pub(crate) fn run( let stop_after_first_crash = args.stop_after_first_crash; // Start executing on this core + let fuzzer = fuzzer.clone(); let thread = std::thread::spawn(move || -> Result<()> { - let result = std::panic::catch_unwind(|| -> Result<()> { + // let mut wrapper = (&mut fuzzer); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| -> Result<()> { start_core::( core_id, + fuzzer, &vm, &project_state.vbcpu, &cpuids, @@ -469,7 +500,7 @@ pub(crate) fn run( #[cfg(feature = "redqueen")] redqueen_breakpoints, ) - }); + })); // Ensure this thread is signalling it is not alive core_stats.lock().unwrap().alive = false; @@ -675,6 +706,7 @@ pub(crate) fn run( /// Thread worker used to fuzz the given [`VbCpu`] state with the given physical memory. fn start_core( core_id: CoreId, + mut fuzzer: FUZZER, vm: &VmFd, vbcpu: &VbCpu, cpuid: &CpuId, @@ -721,8 +753,6 @@ fn start_core( // Unblock SIGALRM to enable this thread to handle SIGALRM unblock_sigalrm()?; - let mut fuzzer = FUZZER::default(); - // RNG for this core used for mutation of inputs let mut rng = Rng::new(); diff --git a/src/fuzzer.rs b/src/fuzzer.rs index b99d000..5e04daa 100644 --- a/src/fuzzer.rs +++ b/src/fuzzer.rs @@ -126,7 +126,7 @@ pub struct Breakpoint { } /// Generic fuzzer trait -pub trait Fuzzer: Default + Sized { +pub trait Fuzzer: Default + Clone + Sized + Send { /// The input type used by this fuzzer type Input: FuzzInput + std::panic::RefUnwindSafe; @@ -154,6 +154,11 @@ pub trait Fuzzer: Default + Sized { fuzzvm: &mut FuzzVm, ) -> Result<()>; + /// Create a new fuzzer - only called once, while per-core fuzzers are cloned. + fn new() -> Self { + Self::default() + } + /// Reset the state of the current fuzzer fn reset_fuzzer_state(&mut self) { // By default, resetting fuzzer state does nothing @@ -322,4 +327,11 @@ pub trait Fuzzer: Default + Sized { fn init_files(&self, _fs: &mut FileSystem) -> Result<()> { Ok(()) } + + + /// Load a seed input (stored by default in `./snapshot/inputs/`) + fn load_seed_input>(&mut self, input_path: P, project_dir: &Path) -> Result> { + let input_path = input_path.as_ref(); + InputWithMetadata::from_path(input_path, project_dir) + } } From 0b0f09deac5df7a78c746996a458b75994123202 Mon Sep 17 00:00:00 2001 From: Michael Rodler Date: Tue, 9 Jan 2024 19:29:29 +0100 Subject: [PATCH 2/2] Fuzzer is now initialized with `new(project_state)` and cloned across all commands. --- src/commands/corpus_min.rs | 9 ++++++--- src/commands/coverage.rs | 2 +- src/commands/find_input.rs | 10 +++++++--- src/commands/fuzz.rs | 2 +- src/commands/minimize.rs | 7 ++++--- src/commands/redqueen.rs | 2 +- src/commands/trace.rs | 2 +- src/fuzzer.rs | 11 +++++++---- 8 files changed, 28 insertions(+), 17 deletions(-) diff --git a/src/commands/corpus_min.rs b/src/commands/corpus_min.rs index 2e6159c..7933e09 100644 --- a/src/commands/corpus_min.rs +++ b/src/commands/corpus_min.rs @@ -117,6 +117,9 @@ pub(crate) fn run( let chunk_size: usize = args.chunk_size; let mut minimizer = CorpusMinimizer::default(); + // create new fuzzer + let mut fuzzer = FUZZER::new(&project_state); + // for chunk in (0..paths.len()).step_by(chunk_size) { let ending_index = (chunk + chunk_size).min(paths.len()); @@ -144,11 +147,13 @@ pub(crate) fn run( let timeout = args.timeout.clone(); let clean_snapshot = clean_snapshot.clone(); let project_dir = project_state.path.clone(); + let fuzzer = fuzzer.clone(); // Start executing on this core let t = std::thread::spawn(move || { start_core::( CoreId { id: core_id }, + fuzzer, &vm, &vbcpu, &cpuids, @@ -292,6 +297,7 @@ pub(crate) fn run( /// Thread worker used to gather coverage for a specific input pub(crate) fn start_core( core_id: CoreId, + mut fuzzer: FUZZER, vm: &VmFd, vbcpu: &VbCpu, cpuid: &CpuId, @@ -317,9 +323,6 @@ pub(crate) fn start_core( // Set the core affinity for this core core_affinity::set_for_current(core_id); - // Use the current fuzzer - let mut fuzzer = FUZZER::default(); - // Sanity check that the given fuzzer matches the snapshot ensure!( FUZZER::START_ADDRESS == vbcpu.rip, diff --git a/src/commands/coverage.rs b/src/commands/coverage.rs index f3784b7..9161810 100644 --- a/src/commands/coverage.rs +++ b/src/commands/coverage.rs @@ -109,7 +109,7 @@ pub(crate) fn start_core( .. } = project_state; // Use the current fuzzer - let mut fuzzer = FUZZER::default(); + let mut fuzzer = FUZZER::new(project_state); // Sanity check that the given fuzzer matches the snapshot ensure!( diff --git a/src/commands/find_input.rs b/src/commands/find_input.rs index 898778f..9f4916f 100644 --- a/src/commands/find_input.rs +++ b/src/commands/find_input.rs @@ -28,6 +28,7 @@ use crate::{Cr3, Execution, ResetBreakpointType, VbCpu, VirtAddr}; #[allow(clippy::needless_pass_by_value)] fn start_core( core_id: CoreId, + mut fuzzer: FUZZER, vm: &VmFd, vbcpu: &VbCpu, cpuid: &CpuId, @@ -53,9 +54,6 @@ fn start_core( // Set the core affinity for this core core_affinity::set_for_current(core_id); - // Create a default fuzzer for single shot, tracing execution with the given input - let mut fuzzer = FUZZER::default(); - // Sanity check that the given fuzzer matches the snapshot ensure!( FUZZER::START_ADDRESS == vbcpu.rip, @@ -239,6 +237,9 @@ pub(crate) fn run( let next_file_index = Arc::new(AtomicUsize::new(0)); let finished = Arc::new(AtomicBool::new(false)); + // create new fuzzer + let mut fuzzer = FUZZER::new(&project_state); + for id in 1..=cores { let files = files.clone(); let physmem_file_fd = physmem_file.as_raw_fd(); @@ -263,10 +264,13 @@ pub(crate) fn run( let clean_snapshot = clean_snapshot.clone(); + let fuzzer = fuzzer.clone(); + // Start executing on this core let t = std::thread::spawn(move || { start_core::( core_id, + fuzzer, &vm, &vbcpu, &cpuids, diff --git a/src/commands/fuzz.rs b/src/commands/fuzz.rs index cfa8080..6f225a1 100644 --- a/src/commands/fuzz.rs +++ b/src/commands/fuzz.rs @@ -107,7 +107,7 @@ pub(crate) fn run( log::warn!("Starting all {} worker threads", cores); - let mut fuzzer = FUZZER::new(); + let mut fuzzer = FUZZER::new(&project_state); // Read the input corpus from the given input directory let mut input_corpus: Vec>> = Vec::new(); diff --git a/src/commands/minimize.rs b/src/commands/minimize.rs index d126c48..fde3593 100644 --- a/src/commands/minimize.rs +++ b/src/commands/minimize.rs @@ -59,10 +59,11 @@ fn start_core( max_iterations: u32, config: Config, min_params: MinimizerConfig, - project_dir: &PathBuf, + project_state: &ProjectState, ) -> Result<()> { + let project_dir = &project_state.path; // Use the current fuzzer - let mut fuzzer = FUZZER::default(); + let mut fuzzer = FUZZER::new(project_state); // Sanity check that the given fuzzer matches the snapshot ensure!( @@ -467,7 +468,7 @@ pub(crate) fn run( args.iterations_per_stage, project_state.config.clone(), minparams, - &project_state.path, + &project_state, )?; // Success diff --git a/src/commands/redqueen.rs b/src/commands/redqueen.rs index a437003..50c13df 100644 --- a/src/commands/redqueen.rs +++ b/src/commands/redqueen.rs @@ -117,7 +117,7 @@ pub(crate) fn start_core( } = project_state; // Use the current fuzzer - let mut fuzzer = FUZZER::default(); + let mut fuzzer = FUZZER::new(project_state); // Sanity check that the given fuzzer matches the snapshot ensure!( diff --git a/src/commands/trace.rs b/src/commands/trace.rs index 14c7492..fcd4145 100644 --- a/src/commands/trace.rs +++ b/src/commands/trace.rs @@ -64,7 +64,7 @@ fn start_core( core_affinity::set_for_current(core_id); // Create a default fuzzer for single shot, tracing execution with the given input - let mut fuzzer = FUZZER::default(); + let mut fuzzer = FUZZER::new(project_state); log::info!("Fuzzer: {:#x} RIP: {:#x}", FUZZER::START_ADDRESS, vbcpu.rip); diff --git a/src/fuzzer.rs b/src/fuzzer.rs index 5e04daa..81214cb 100644 --- a/src/fuzzer.rs +++ b/src/fuzzer.rs @@ -15,7 +15,6 @@ use std::sync::{Arc, RwLock}; use crate::addrs::{Cr3, VirtAddr}; use crate::cmp_analysis::RedqueenRule; -use crate::expensive_mutators; use crate::feedback::FeedbackTracker; use crate::filesystem::FileSystem; use crate::fuzz_input::{FuzzInput, InputMetadata, InputWithMetadata}; @@ -23,6 +22,7 @@ use crate::fuzzvm::{FuzzVm, HookFn}; use crate::mutators; use crate::rng::Rng; use crate::Execution; +use crate::{expensive_mutators, ProjectState}; /// Type of breakpoint being applied #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] @@ -155,7 +155,7 @@ pub trait Fuzzer: Default + Clone + Sized + Send { ) -> Result<()>; /// Create a new fuzzer - only called once, while per-core fuzzers are cloned. - fn new() -> Self { + fn new(project_state: &ProjectState) -> Self { Self::default() } @@ -328,9 +328,12 @@ pub trait Fuzzer: Default + Clone + Sized + Send { Ok(()) } - /// Load a seed input (stored by default in `./snapshot/inputs/`) - fn load_seed_input>(&mut self, input_path: P, project_dir: &Path) -> Result> { + fn load_seed_input>( + &mut self, + input_path: P, + project_dir: &Path, + ) -> Result> { let input_path = input_path.as_ref(); InputWithMetadata::from_path(input_path, project_dir) }