From dc31ffc92bb1e1d2bf3929b7648af20d4ba5146d Mon Sep 17 00:00:00 2001 From: bmc-msft <41130664+bmc-msft@users.noreply.github.com> Date: Fri, 22 Jan 2021 18:20:22 -0500 Subject: [PATCH] add support for fully self-contained fuzzers (#454) --- .../src/local/generic_crash_report.rs | 2 +- .../src/local/libfuzzer_crash_report.rs | 2 +- .../src/tasks/fuzz/supervisor.rs | 185 ++++++++++-------- .../src/tasks/report/crash_report.rs | 55 +++++- .../onefuzz-agent/src/tasks/report/generic.rs | 2 +- .../src/tasks/report/libfuzzer_report.rs | 6 +- src/agent/onefuzz/src/expand.rs | 9 + src/agent/onefuzz/src/syncdir.rs | 2 +- 8 files changed, 171 insertions(+), 92 deletions(-) diff --git a/src/agent/onefuzz-agent/src/local/generic_crash_report.rs b/src/agent/onefuzz-agent/src/local/generic_crash_report.rs index 4ef3d976c0..fb19eb174c 100644 --- a/src/agent/onefuzz-agent/src/local/generic_crash_report.rs +++ b/src/agent/onefuzz-agent/src/local/generic_crash_report.rs @@ -29,7 +29,7 @@ pub fn build_report_config(args: &clap::ArgMatches<'_>) -> Result { } else { None }; - let unique_reports = value_t!(args, UNIQUE_REPORTS_DIR, PathBuf)?.into(); + let unique_reports = Some(value_t!(args, UNIQUE_REPORTS_DIR, PathBuf)?.into()); let target_timeout = value_t!(args, TARGET_TIMEOUT, u64).ok(); diff --git a/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs b/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs index 7a8a9ee8b5..a233b07d72 100644 --- a/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs +++ b/src/agent/onefuzz-agent/src/local/libfuzzer_crash_report.rs @@ -29,7 +29,7 @@ pub fn build_report_config(args: &clap::ArgMatches<'_>) -> Result { } else { None }; - let unique_reports = value_t!(args, UNIQUE_REPORTS_DIR, PathBuf)?.into(); + let unique_reports = Some(value_t!(args, UNIQUE_REPORTS_DIR, PathBuf)?.into()); let target_timeout = value_t!(args, TARGET_TIMEOUT, u64).ok(); diff --git a/src/agent/onefuzz-agent/src/tasks/fuzz/supervisor.rs b/src/agent/onefuzz-agent/src/tasks/fuzz/supervisor.rs index 37dab704f0..2b118c8f11 100644 --- a/src/agent/onefuzz-agent/src/tasks/fuzz/supervisor.rs +++ b/src/agent/onefuzz-agent/src/tasks/fuzz/supervisor.rs @@ -5,6 +5,7 @@ use crate::tasks::{ config::{CommonConfig, ContainerType}, heartbeat::*, + report::crash_report::monitor_reports, stats::common::{monitor_stats, StatsFormat}, utils::CheckNotify, }; @@ -24,12 +25,13 @@ use std::{ process::Stdio, time::Duration, }; +use tempfile::tempdir; use tokio::{ process::{Child, Command}, sync::Notify, }; -#[derive(Debug, Deserialize)] +#[derive(Debug, Deserialize, Default)] pub struct SupervisorConfig { pub inputs: SyncedDir, pub crashes: SyncedDir, @@ -37,13 +39,16 @@ pub struct SupervisorConfig { pub supervisor_env: HashMap, pub supervisor_options: Vec, pub supervisor_input_marker: Option, - pub target_exe: PathBuf, - pub target_options: Vec, - pub tools: SyncedDir, + pub target_exe: Option, + pub target_options: Option>, + pub tools: Option, pub wait_for_files: Option, pub stats_file: Option, pub stats_format: Option, pub ensemble_sync_delay: Option, + pub reports: Option, + pub unique_reports: Option, + pub no_repro: Option, #[serde(flatten)] pub common: CommonConfig, } @@ -54,27 +59,43 @@ pub async fn spawn(config: SupervisorConfig) -> Result<(), Error> { let runtime_dir = OwnedDir::new(config.common.task_id.to_string()); runtime_dir.create_if_missing().await?; - config.tools.init_pull().await?; - set_executable(&config.tools.path).await?; - - let supervisor_path = Expand::new() - .tools_dir(&config.tools.path) - .evaluate_value(&config.supervisor_exe)?; + // setup tools + if let Some(tools) = &config.tools { + tools.init_pull().await?; + set_executable(&tools.path).await?; + } + // setup crashes let crashes = SyncedDir { path: runtime_dir.path().join("crashes"), url: config.crashes.url.clone(), }; - crashes.init().await?; let monitor_crashes = crashes.monitor_results(new_result); + // setup reports + let reports_dir = tempdir()?; + if let Some(unique_reports) = &config.unique_reports { + unique_reports.init().await?; + } + if let Some(reports) = &config.reports { + reports.init().await?; + } + if let Some(no_repro) = &config.no_repro { + no_repro.init().await?; + } + let monitor_reports_future = monitor_reports( + reports_dir.path(), + &config.unique_reports, + &config.reports, + &config.no_repro, + ); + let inputs = SyncedDir { path: runtime_dir.path().join("inputs"), url: config.inputs.url.clone(), }; inputs.init().await?; - if let Some(context) = &config.wait_for_files { let dir = match context { ContainerType::Inputs => &inputs, @@ -94,16 +115,10 @@ pub async fn spawn(config: SupervisorConfig) -> Result<(), Error> { let process = start_supervisor( &runtime_dir.path(), - &supervisor_path, - &config.target_exe, - &crashes.path, - &inputs.path, - &config.target_options, - &config.supervisor_options, - &config.supervisor_env, - &config.supervisor_input_marker, - &config.common.setup_dir, - &config.tools.path, + &config, + &crashes, + &inputs, + reports_dir.path().to_path_buf(), ) .await?; @@ -133,6 +148,7 @@ pub async fn spawn(config: SupervisorConfig) -> Result<(), Error> { monitor_stats, monitor_crashes, continuous_sync_task, + monitor_reports_future, )?; Ok(()) @@ -150,45 +166,49 @@ async fn heartbeat_process( async fn start_supervisor( runtime_dir: impl AsRef, - supervisor_path: impl AsRef, - target_exe: impl AsRef, - fault_dir: impl AsRef, - inputs_dir: impl AsRef, - target_options: &[String], - supervisor_options: &[String], - supervisor_env: &HashMap, - supervisor_input_marker: &Option, - setup_dir: impl AsRef, - tools_dir: impl AsRef, + config: &SupervisorConfig, + crashes: &SyncedDir, + inputs: &SyncedDir, + reports_dir: PathBuf, ) -> Result { - let mut cmd = Command::new(supervisor_path.as_ref()); + let mut expand = Expand::new(); + expand + .supervisor_exe(&config.supervisor_exe) + .supervisor_options(&config.supervisor_options) + .runtime_dir(&runtime_dir) + .crashes(&crashes.path) + .input_corpus(&inputs.path) + .reports_dir(&reports_dir) + .setup_dir(&config.common.setup_dir); + + if let Some(tools) = &config.tools { + expand.tools_dir(&tools.path); + } + + if let Some(target_exe) = &config.target_exe { + expand.target_exe(target_exe); + } + if let Some(target_options) = &config.target_options { + expand.target_options(target_options); + } + + if let Some(input_marker) = &config.supervisor_input_marker { + expand.input_marker(input_marker); + } + + let supervisor_path = expand.evaluate_value(&config.supervisor_exe)?; + let mut cmd = Command::new(supervisor_path); let cmd = cmd .kill_on_drop(true) .env_remove("RUST_LOG") .stdout(Stdio::piped()) .stderr(Stdio::piped()); - let mut expand = Expand::new(); - expand - .supervisor_exe(supervisor_path) - .supervisor_options(supervisor_options) - .crashes(fault_dir) - .runtime_dir(runtime_dir) - .target_exe(target_exe) - .target_options(target_options) - .input_corpus(inputs_dir) - .setup_dir(setup_dir) - .tools_dir(tools_dir); - - if let Some(input_marker) = supervisor_input_marker { - expand.input_marker(input_marker); - } - - let args = expand.evaluate(supervisor_options)?; + let args = expand.evaluate(&config.supervisor_options)?; cmd.args(&args); - for (k, v) in supervisor_env { + for (k, v) in &config.supervisor_env { cmd.env(k, expand.evaluate_value(v)?); } @@ -228,26 +248,39 @@ mod tests { use std::env; let runtime_dir = tempfile::tempdir().unwrap(); - let afl_fuzz_exe = if let Ok(x) = env::var("ONEFUZZ_TEST_AFL_LINUX_FUZZER") { - x + + let supervisor_exe = if let Ok(x) = env::var("ONEFUZZ_TEST_AFL_LINUX_FUZZER") { + x.into() } else { warn!("Unable to test AFL integration"); return; }; - let afl_test_binary = if let Ok(x) = env::var("ONEFUZZ_TEST_AFL_LINUX_TEST_BINARY") { - x + let target_exe = if let Ok(x) = env::var("ONEFUZZ_TEST_AFL_LINUX_TEST_BINARY") { + Some(x.into()) } else { warn!("Unable to test AFL integration"); return; }; + let reports_dir_temp = tempfile::tempdir().unwrap(); + let reports_dir = reports_dir_temp.path().into(); + let fault_dir_temp = tempfile::tempdir().unwrap(); - let fault_dir = fault_dir_temp.path(); + let crashes = SyncedDir { + path: fault_dir_temp.path().into(), + url: None, + }; + let corpus_dir_temp = tempfile::tempdir().unwrap(); - let corpus_dir = corpus_dir_temp.into_path(); - let seed_file_name = corpus_dir.clone().join("seed.txt"); - let target_options = vec!["{input}".to_owned()]; + let corpus_dir = SyncedDir { + path: corpus_dir_temp.path().into(), + url: None, + }; + let seed_file_name = corpus_dir.path.join("seed.txt"); + tokio::fs::write(seed_file_name, "xyz").await.unwrap(); + + let target_options = Some(vec!["{input}".to_owned()]); let supervisor_env = HashMap::new(); let supervisor_options: Vec<_> = vec![ "-d", @@ -266,32 +299,24 @@ mod tests { // AFL input marker let supervisor_input_marker = Some("@@".to_owned()); - println!( - "testing 2: corpus_dir {:?} -- fault_dir {:?} -- seed_file_name {:?}", - corpus_dir, fault_dir, seed_file_name - ); + let config = SupervisorConfig { + supervisor_exe, + supervisor_env, + supervisor_options, + supervisor_input_marker, + target_exe, + target_options, + ..Default::default() + }; - tokio::fs::write(seed_file_name, "xyz").await.unwrap(); - let process = start_supervisor( - runtime_dir, - PathBuf::from(afl_fuzz_exe), - PathBuf::from(afl_test_binary), - fault_dir, - corpus_dir, - &target_options, - &supervisor_options, - &supervisor_env, - &supervisor_input_marker, - PathBuf::default(), - PathBuf::default(), - ) - .await - .unwrap(); + let process = start_supervisor(runtime_dir, &config, &crashes, &corpus_dir, reports_dir) + .await + .unwrap(); let notify = Notify::new(); let _fuzzing_monitor = monitor_process(process, "supervisor".to_string(), false, Some(¬ify)); - let stat_output = fault_dir.join("fuzzer_stats"); + let stat_output = crashes.path.join("fuzzer_stats"); let start = Instant::now(); loop { if has_stats(&stat_output).await { diff --git a/src/agent/onefuzz-agent/src/tasks/report/crash_report.rs b/src/agent/onefuzz-agent/src/tasks/report/crash_report.rs index cdc39df792..4bb993daf5 100644 --- a/src/agent/onefuzz-agent/src/tasks/report/crash_report.rs +++ b/src/agent/onefuzz-agent/src/tasks/report/crash_report.rs @@ -1,11 +1,13 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -use anyhow::Result; +use anyhow::{Context, Result}; +use futures::StreamExt; use onefuzz::{ asan::AsanLog, blob::{BlobClient, BlobUrl}, fs::exists, + monitor::DirectoryMonitor, syncdir::SyncedDir, telemetry::{ Event::{new_report, new_unable_to_reproduce, new_unique_report}, @@ -15,7 +17,7 @@ use onefuzz::{ use reqwest::{StatusCode, Url}; use reqwest_retry::SendRetry; use serde::{Deserialize, Serialize}; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use tokio::fs; use uuid::Uuid; @@ -104,16 +106,18 @@ async fn upload_or_save_local( impl CrashTestResult { pub async fn save( &self, - unique_reports: &SyncedDir, + unique_reports: &Option, reports: &Option, no_repro: &Option, ) -> Result<()> { match self { Self::CrashReport(report) => { // Use SHA-256 of call stack as dedupe key. - let name = report.unique_blob_name(); - if upload_or_save_local(&report, &name, unique_reports).await? { - event!(new_unique_report; EventData::Path = name); + if let Some(unique_reports) = unique_reports { + let name = report.unique_blob_name(); + if upload_or_save_local(&report, &name, unique_reports).await? { + event!(new_unique_report; EventData::Path = name); + } } if let Some(reports) = reports { @@ -193,3 +197,42 @@ impl NoCrash { format!("{}.json", self.input_sha256) } } + +async fn parse_report_file(path: PathBuf) -> Result { + let raw = std::fs::read_to_string(&path) + .with_context(|| format_err!("unable to open crash report: {}", path.display()))?; + + let json: serde_json::Value = serde_json::from_str(&raw) + .with_context(|| format_err!("invalid json: {} - {:?}", path.display(), raw))?; + + let report: Result = serde_json::from_value(json.clone()); + if let Ok(report) = report { + return Ok(CrashTestResult::CrashReport(report)); + } + let no_repro: Result = serde_json::from_value(json); + if let Ok(no_repro) = no_repro { + return Ok(CrashTestResult::NoRepro(no_repro)); + } + + bail!("unable to parse report: {} - {:?}", path.display(), raw) +} + +pub async fn monitor_reports( + base_dir: &Path, + unique_reports: &Option, + reports: &Option, + no_crash: &Option, +) -> Result<()> { + if unique_reports.is_none() && reports.is_none() && no_crash.is_none() { + verbose!("no report directories configured"); + return Ok(()); + } + + let mut monitor = DirectoryMonitor::new(base_dir); + monitor.start()?; + while let Some(file) = monitor.next().await { + let result = parse_report_file(file).await?; + result.save(unique_reports, reports, no_crash).await?; + } + Ok(()) +} diff --git a/src/agent/onefuzz-agent/src/tasks/report/generic.rs b/src/agent/onefuzz-agent/src/tasks/report/generic.rs index a69650b076..db1893ac63 100644 --- a/src/agent/onefuzz-agent/src/tasks/report/generic.rs +++ b/src/agent/onefuzz-agent/src/tasks/report/generic.rs @@ -35,7 +35,7 @@ pub struct Config { pub input_queue: Option, pub crashes: Option, pub reports: Option, - pub unique_reports: SyncedDir, + pub unique_reports: Option, pub no_repro: Option, pub target_timeout: Option, diff --git a/src/agent/onefuzz-agent/src/tasks/report/libfuzzer_report.rs b/src/agent/onefuzz-agent/src/tasks/report/libfuzzer_report.rs index 9903c63576..1000f0db76 100644 --- a/src/agent/onefuzz-agent/src/tasks/report/libfuzzer_report.rs +++ b/src/agent/onefuzz-agent/src/tasks/report/libfuzzer_report.rs @@ -30,7 +30,7 @@ pub struct Config { pub input_queue: Option, pub crashes: Option, pub reports: Option, - pub unique_reports: SyncedDir, + pub unique_reports: Option, pub no_repro: Option, #[serde(default = "default_bool_true")] @@ -67,7 +67,9 @@ impl ReportTask { }; crashes.init().await?; - self.config.unique_reports.init().await?; + if let Some(unique_reports) = &self.config.unique_reports { + unique_reports.init().await?; + } if let Some(reports) = &self.config.reports { reports.init().await?; } diff --git a/src/agent/onefuzz/src/expand.rs b/src/agent/onefuzz/src/expand.rs index 7c537f780c..8e6fcfbf12 100644 --- a/src/agent/onefuzz/src/expand.rs +++ b/src/agent/onefuzz/src/expand.rs @@ -34,6 +34,7 @@ pub enum PlaceHolder { SupervisorExe, SupervisorOptions, SetupDir, + ReportsDir, } impl PlaceHolder { @@ -57,6 +58,7 @@ impl PlaceHolder { Self::SupervisorExe => "{supervisor_exe}", Self::SupervisorOptions => "{supervisor_options}", Self::SetupDir => "{setup_dir}", + Self::ReportsDir => "{reports_dir}", } .to_string() } @@ -203,6 +205,13 @@ impl<'a> Expand<'a> { self } + pub fn reports_dir(&mut self, arg: impl AsRef) -> &mut Self { + let arg = arg.as_ref(); + let path = String::from(arg.to_string_lossy()); + self.set_value(PlaceHolder::ReportsDir, ExpandedValue::Path(path)); + self + } + pub fn tools_dir(&mut self, arg: impl AsRef) -> &mut Self { let arg = arg.as_ref(); let path = String::from(arg.to_string_lossy()); diff --git a/src/agent/onefuzz/src/syncdir.rs b/src/agent/onefuzz/src/syncdir.rs index 1b9aa003ad..b9e2cf7e5a 100644 --- a/src/agent/onefuzz/src/syncdir.rs +++ b/src/agent/onefuzz/src/syncdir.rs @@ -23,7 +23,7 @@ pub enum SyncOperation { const DELAY: Duration = Duration::from_secs(10); const DEFAULT_CONTINUOUS_SYNC_DELAY_SECONDS: u64 = 60; -#[derive(Debug, Deserialize, Clone, PartialEq)] +#[derive(Debug, Deserialize, Clone, PartialEq, Default)] pub struct SyncedDir { pub path: PathBuf, pub url: Option,