libafl-fuzz: separate frida build + cmplog debug (#2591)

* libafl-fuzz: separate frida build

* cmplog debug

* update

* merge AflStatsStage
move time_tracker stage to LibAFL

* mandate track_hit_feedbacks feature for AflStatsStage

* afl_stats do not hardcode TimeoutFeedback and CrashFeedback names

* typo

* typo

* fix generics order

* add verify timeouts stage

* libafl: introduce set_timeout func to dynamically set timeouts for executor
libafl-fuzz: add verify_timeout stage

* add missing set_timeout implementations

* libafl-fuzz: move set_timeout and timeout from Executor to HasTimeout

* libafl-fuzz: add removed gitignore

* remove timeout from libafl_nyx::Executor and move it to NyxHelper

* clippy

* fix HasTimeout for QemuExecutor

* libafl-fuzz: remove observer handle usage in verify_timeouts
misc: remove prelude imports

* libafl-fuzz: fix foreign_sync_dirs option

* fmt && clippy

* clippy && fmt

* missing doc

* clippy

* bruh

* damned doc build

* trait fix

* impl HasTimeout for InProcessExecutor only if std

* clippy

* fix typo

* fix nostd build

* clippy

* remove most HasTimeout implementations for now

* typo

* remove redundant import

* misc

* fmt

* simplify trait bounds

* add old AflStatsStage back and rename it to StatsStage

* fix ci

* make set_timeout and timeout of HasTimeout inline

* fmt

* add gitignore

* serde_any fix

* tmate

* misc

* remove tmate

* test

* coordinate between capture_timeout and verify_timeout

* makefile

* fix

* fix

* fmt

* increase cmplog timeout

* semantic

* debug

* debug

* remove dbeug

* only test libafl-fuzz on CI for now

* better seed for cmplog?

* remove preflight check for now

* set Input type in forkserver

* debug

* tmate

* fix capture_timeout

* revert workflow

* run only libafl-fuzz

* remove pre-flight

* re-enable fuzzers on CI

* move capture_timeouts and verify_timeouts to main lib

* run fmt

* add note for verify timeouts

* add note in verify timeouts stage

* typo

---------

Co-authored-by: Dominik Maier <domenukk@gmail.com>

Author: R9295

fuzzers/baby/backtrace_baby_fuzzers/command_executor/src/main.rs

@@ -84,6 +84,7 @@ pub fn main() {
     #[derive(Debug)]
     struct MyExecutor {
         shmem_id: ShMemId,
+        timeout: Duration,
     }
 
     impl CommandConfigurator<BytesInput> for MyExecutor {
@@ -106,11 +107,16 @@ pub fn main() {
         }
 
         fn exec_timeout(&self) -> Duration {
-            Duration::from_secs(5)
+            self.timeout
+        }
+        fn exec_timeout_mut(&mut self) -> &mut Duration {
+            &mut self.timeout
         }
     }
 
-    let mut executor = MyExecutor { shmem_id }.into_executor(tuple_list!(observer, bt_observer));
+    let timeout = Duration::from_secs(5);
+    let mut executor =
+        MyExecutor { shmem_id, timeout }.into_executor(tuple_list!(observer, bt_observer));
 
     // Generator of printable bytearrays of max size 32
     let mut generator = RandPrintablesGenerator::new(nonzero!(32));

fuzzers/binary_only/frida_libpng/harness_win.cpp

@@ -4,7 +4,7 @@
 #include <string.h>
 
 extern "C" __declspec(dllexport) size_t
-LLVMFuzzerTestOneInput(const char *data, unsigned int len) {
+    LLVMFuzzerTestOneInput(const char *data, unsigned int len) {
   if (data[0] == 'b') {
     if (data[1] == 'a') {
       if (data[2] == 'd') {

fuzzers/binary_only/qemu_launcher/src/instance.rs

@@ -23,8 +23,8 @@ use libafl::{
         powersched::PowerSchedule, IndexesLenTimeMinimizerScheduler, PowerQueueScheduler,
     },
     stages::{
-        calibrate::CalibrationStage, power::StdPowerMutationalStage, AflStatsStage, IfStage,
-        ShadowTracingStage, StagesTuple, StdMutationalStage,
+        calibrate::CalibrationStage, power::StdPowerMutationalStage, IfStage, ShadowTracingStage,
+        StagesTuple, StatsStage, StdMutationalStage,
     },
     state::{HasCorpus, StdState, UsesState},
     Error, HasMetadata, NopFuzzer,
@@ -138,7 +138,7 @@ impl<M: Monitor> Instance<'_, M> {
 
         let stats_stage = IfStage::new(
             |_, _, _, _| Ok(self.options.tui),
-            tuple_list!(AflStatsStage::new(Duration::from_secs(5))),
+            tuple_list!(StatsStage::new(Duration::from_secs(5))),
         );
 
         // Feedback to rate the interestingness of an input

fuzzers/forkserver/libafl-fuzz/Makefile.toml

@@ -12,7 +12,7 @@ FUZZER = '${CARGO_TARGET_DIR}/${PROFILE_DIR}/${FUZZER_NAME}'
 LLVM_CONFIG = { value = "llvm-config-18", condition = { env_not_set = [
   "LLVM_CONFIG",
 ] } }
-AFL_VERSION = "8b35dd49be5f846e945f6d6a9414623d195a99cb"
+AFL_VERSION = "78b7e14c73baacf1d88b3c03955e78f5080d17ba"
 AFL_DIR = { value = "${PROJECT_DIR}/AFLplusplus" }
 AFL_CC_PATH = { value = "${AFL_DIR}/afl-clang-fast" }
 CC = { value = "clang" }
@@ -25,12 +25,16 @@ if [ ! -d "$AFL_DIR" ]; then
 	cd ${AFL_DIR}
 	git checkout ${AFL_VERSION}
 	LLVM_CONFIG=${LLVM_CONFIG} make 
+fi
+'''
+[tasks.build_frida_mode]
+script_runner = '@shell'
+script = '''
+	cd ${AFL_DIR}
 	cd frida_mode
 	LLVM_CONFIG=${LLVM_CONFIG} make
 	cd ../..
-fi
 '''
-
 [tasks.build_qemuafl]
 script_runner = "@shell"
 script = '''
@@ -77,7 +81,7 @@ script = '''
 AFL_PATH=${AFL_DIR} ${AFL_CC_PATH} ./test/test-instr.c -o ./test/out-instr
 
 export LIBAFL_DEBUG_OUTPUT=1 
-export AFL_CORES=1 
+export AFL_CORES=0
 export AFL_STATS_INTERVAL=1 
 
 timeout 5 ${FUZZER} -i ./test/seeds -o ./test/output ./test/out-instr || true
@@ -109,7 +113,7 @@ script_runner = "@shell"
 script = '''
 # cmplog TODO: AFL_BENCH_UNTIL_CRASH=1 instead of timeout 15s
 AFL_LLVM_CMPLOG=1 AFL_PATH=${AFL_DIR} ${AFL_CC_PATH} ./test/test-cmplog.c -o ./test/out-cmplog
-AFL_CORES=1 timeout 5 ${FUZZER} -Z -l 3 -m 0 -V30 -i ./test/seeds_cmplog -o ./test/output-cmplog -c 0 ./test/out-cmplog || true
+LIBAFL_DEBUG_OUTPUT=1 AFL_CORES=0 timeout 10 ${FUZZER} -Z -l 3 -m 0 -V30 -i ./test/seeds_cmplog -o ./test/output-cmplog -c 0 ./test/out-cmplog || true
 test -n "$( ls ${PROJECT_DIR}/test/output-cmplog/fuzzer_main/hangs/id:0000* ${PROJECT_DIR}/test/output-cmplog/fuzzer_main/crashes/id:0000*)" || {
 	echo "No crashes found"
 	exit 1
@@ -123,7 +127,7 @@ script = '''
 ${CC} -no-pie ./test/test-instr.c -o ./test/out-frida
 
 export AFL_PATH=${AFL_DIR} 
-export AFL_CORES=1 
+export AFL_CORES=0
 export AFL_STATS_INTERVAL=1 
 
 timeout 5 ${FUZZER} -m 0  -O -i ./test/seeds_frida -o ./test/output-frida -- ./test/out-frida || true
@@ -162,7 +166,7 @@ test -n "$RUNTIME" -a -n "$RUNTIME_PERSISTENT" && {
 
 unset AFL_FRIDA_PERSISTENT_ADDR
 '''
-dependencies = ["build_afl", "build_libafl_fuzz"]
+dependencies = ["build_afl", "build_frida_mode", "build_libafl_fuzz"]
 
 [tasks.test_qemu]
 script_runner = "@shell"
@@ -171,7 +175,7 @@ ${CC} -pie -fPIE ./test/test-instr.c -o ./test/out-qemu
 ${CC} -o ./test/out-qemu-cmpcov ./test/test-cmpcov.c
 
 export AFL_PATH=${AFL_DIR}
-export AFL_CORES=1
+export AFL_CORES=0
 export AFL_STATS_INTERVAL=1
 
 timeout 5 ${FUZZER} -m 0  -Q -i ./test/seeds_qemu -o ./test/output-qemu -- ./test/out-qemu || true
@@ -202,7 +206,7 @@ dependencies = ["build_afl", "build_qemuafl", "build_libafl_fuzz"]
 script_runner = "@shell"
 script = '''
 export AFL_PATH=${AFL_DIR}
-export AFL_CORES=1
+export AFL_CORES=0
 export AFL_STATS_INTERVAL=1
 
 # TODO: test unicorn persistent mode once it's fixed on AFL++

fuzzers/forkserver/libafl-fuzz/src/corpus.rs

@@ -128,7 +128,7 @@ pub fn check_autoresume(fuzzer_dir: &Path, auto_resume: bool) -> Result<Flock<Fi
             }
         }
         if !auto_resume && last_update.saturating_sub(start_time) > OUTPUT_GRACE * 60 {
-            return Err(Error::illegal_state("The job output directory already exists and contains results! use AFL_AUTORESUME=true or provide \"-\" for -i "));
+            return Err(Error::illegal_state("The job output directory already exists and contains results! use AFL_AUTORESUME=1 or provide \"-\" for -i "));
         }
     }
     if !auto_resume {

fuzzers/forkserver/libafl-fuzz/src/env_parser.rs

@@ -1,6 +1,6 @@
 use std::{collections::HashMap, path::PathBuf, time::Duration};
 
-use libafl::Error;
+use libafl::{stages::afl_stats::AFL_FUZZER_STATS_UPDATE_INTERVAL_SECS, Error};
 use libafl_bolts::core_affinity::Cores;
 
 use crate::Opt;
@@ -73,6 +73,8 @@ pub fn parse_envs(opt: &mut Opt) -> Result<(), Error> {
     }
     if let Ok(res) = std::env::var("AFL_FUZZER_STATS_UPDATE_INTERVAL") {
         opt.stats_interval = res.parse()?;
+    } else {
+        opt.stats_interval = AFL_FUZZER_STATS_UPDATE_INTERVAL_SECS;
     }
     if let Ok(res) = std::env::var("AFL_BROKER_PORT") {
         opt.broker_port = Some(res.parse()?);

fuzzers/forkserver/libafl-fuzz/src/feedback/seed.rs

@@ -65,7 +65,7 @@ where
                     if !self.ignore_timeouts {
                         if !self.ignore_seed_issues || self.exit_on_seed_issues {
                             return Err(Error::invalid_corpus(
-                                "input led to a timeout; use AFL_IGNORE_SEED_ISSUES=true",
+                                "input led to a timeout; use AFL_IGNORE_SEED_ISSUES=1",
                             ));
                         }
                         return Ok(false);

fuzzers/forkserver/libafl-fuzz/src/fuzzer.rs

@@ -1,7 +1,9 @@
 use std::{
     borrow::Cow,
+    cell::RefCell,
     marker::PhantomData,
     path::{Path, PathBuf},
+    rc::Rc,
     time::Duration,
 };
 
@@ -13,7 +15,9 @@ use libafl::{
     },
     executors::forkserver::{ForkserverExecutor, ForkserverExecutorBuilder},
     feedback_and, feedback_or, feedback_or_fast,
-    feedbacks::{ConstFeedback, CrashFeedback, MaxMapFeedback, TimeFeedback, TimeoutFeedback},
+    feedbacks::{
+        CaptureTimeoutFeedback, ConstFeedback, CrashFeedback, MaxMapFeedback, TimeFeedback,
+    },
     fuzzer::StdFuzzer,
     inputs::{BytesInput, NopTargetBytesConverter},
     mutators::{havoc_mutations, tokens_mutations, AFLppRedQueen, StdScheduledMutator, Tokens},
@@ -23,8 +27,11 @@ use libafl::{
         IndexesLenTimeMinimizerScheduler, QueueScheduler, StdWeightedScheduler,
     },
     stages::{
-        mutational::MultiMutationalStage, CalibrationStage, ColorizationStage, IfStage,
-        StagesTuple, StdMutationalStage, StdPowerMutationalStage, SyncFromDiskStage,
+        afl_stats::{AflStatsStage, CalibrationTime, FuzzTime, SyncTime},
+        mutational::MultiMutationalStage,
+        time_tracker::TimeTrackingStageWrapper,
+        CalibrationStage, ColorizationStage, IfStage, StagesTuple, StdMutationalStage,
+        StdPowerMutationalStage, SyncFromDiskStage, VerifyTimeoutsStage,
     },
     state::{
         HasCorpus, HasCurrentTestcase, HasExecutions, HasLastReportTime, HasStartTime, StdState,
@@ -46,7 +53,6 @@ use libafl_targets::{cmps::AFLppCmpLogMap, AFLppCmpLogObserver, AFLppCmplogTraci
 use serde::{Deserialize, Serialize};
 
 use crate::{
-    afl_stats::{AflStatsStage, CalibrationTime, FuzzTime, SyncTime},
     corpus::{set_corpus_filepath, set_solution_filepath},
     env_parser::AFL_DEFAULT_MAP_SIZE,
     executor::find_afl_binary,
@@ -55,7 +61,7 @@ use crate::{
         seed::SeedFeedback,
     },
     scheduler::SupportedSchedulers,
-    stages::{mutational_stage::SupportedMutationalStages, time_tracker::TimeTrackingStageWrapper},
+    stages::mutational_stage::SupportedMutationalStages,
     Opt, AFL_DEFAULT_INPUT_LEN_MAX, AFL_DEFAULT_INPUT_LEN_MIN, AFL_HARNESS_FILE_INPUT,
     SHMEM_ENV_VAR,
 };
@@ -109,17 +115,21 @@ where
     let mut tokens = Tokens::new();
     tokens = tokens.add_from_files(&opt.dicts)?;
 
-    let user_token_count = tokens.len();
-
     // Create a AFLStatsStage;
-    let afl_stats_stage = AflStatsStage::new(
-        opt,
-        fuzzer_dir.to_path_buf(),
-        &edges_observer,
-        user_token_count,
-        !opt.no_autodict,
-        core_id,
-    );
+    let afl_stats_stage = AflStatsStage::builder()
+        .stats_file(fuzzer_dir.join("fuzzer_stats"))
+        .plot_file(fuzzer_dir.join("plot_data"))
+        .core_id(core_id)
+        .report_interval(Duration::from_secs(opt.stats_interval))
+        .map_observer(&edges_observer)
+        .uses_autotokens(!opt.no_autodict)
+        .tokens(&tokens)
+        .banner(opt.executable.display().to_string())
+        .version("0.13.2".to_string())
+        .exec_timeout(opt.hang_timeout)
+        .target_mode(fuzzer_target_mode(opt).to_string())
+        .build()
+        .expect("invariant; should never occur");
 
     // Create an observation channel to keep track of the execution time.
     let time_observer = TimeObserver::new("time");
@@ -140,6 +150,20 @@ where
         opt,
     );
 
+    // We need to share this reference as [`VerifyTimeoutsStage`] will toggle this
+    // value before re-running the alleged timeouts so we don't keep capturing timeouts infinitely.
+    let enable_capture_timeouts = Rc::new(RefCell::new(false));
+    let capture_timeout_feedback = CaptureTimeoutFeedback::new(Rc::clone(&enable_capture_timeouts));
+
+    // Like AFL++ we re-run all timeouts with double the timeout to assert that they are not false positives
+    let timeout_verify_stage = IfStage::new(
+        |_, _, _, _| Ok(!opt.ignore_timeouts),
+        tuple_list!(VerifyTimeoutsStage::new(
+            enable_capture_timeouts,
+            Duration::from_millis(opt.hang_timeout),
+        )),
+    );
+
     /*
      * Feedback to decide if the Input is "solution worthy".
      * We check if it's a crash or a timeout (if we are configured to consider timeouts)
@@ -153,7 +177,7 @@ where
                 CrashFeedback::new(),
                 feedback_and!(
                     ConstFeedback::new(!opt.ignore_timeouts),
-                    TimeoutFeedback::new()
+                    capture_timeout_feedback,
                 )
             ),
             MaxMapFeedback::with_name("edges_objective", &edges_observer)
@@ -396,6 +420,7 @@ where
             calibration,
             cmplog,
             mutational_stage,
+            timeout_verify_stage,
             afl_stats_stage,
             sync_stage
         );
@@ -411,7 +436,13 @@ where
         )?;
     } else {
         // The order of the stages matter!
-        let mut stages = tuple_list!(calibration, mutational_stage, afl_stats_stage, sync_stage);
+        let mut stages = tuple_list!(
+            calibration,
+            mutational_stage,
+            timeout_verify_stage,
+            afl_stats_stage,
+            sync_stage
+        );
 
         // Run our fuzzer; NO CmpLog
         run_fuzzer_with_stages(

fuzzers/forkserver/libafl-fuzz/src/main.rs

@@ -66,7 +66,6 @@
 )]
 
 use std::{collections::HashMap, path::PathBuf, time::Duration};
-mod afl_stats;
 mod env_parser;
 mod feedback;
 mod scheduler;
@@ -188,7 +187,7 @@ struct Opt {
     #[arg(short = 'c')]
     cmplog: Option<String>,
     /// sync to a foreign fuzzer queue directory (requires -M, can be specified up to 32 times)
-    #[arg(short = 'F', num_args = 32)]
+    #[arg(short = 'F')]
     foreign_sync_dirs: Vec<PathBuf>,
     /// fuzzer dictionary (see README.md)
     #[arg(short = 'x')]

fuzzers/forkserver/libafl-fuzz/src/stages/mod.rs

@@ -1,2 +1 @@
 pub mod mutational_stage;
-pub mod time_tracker;