From 169d6c16fd25e4387d4b68767ad8a05ecb94f030 Mon Sep 17 00:00:00 2001 From: epi Date: Tue, 10 Nov 2020 06:18:20 -0600 Subject: [PATCH 01/18] added normalize_url to utils --- src/utils.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/utils.rs b/src/utils.rs index e70a8f87..ec66f0e2 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -312,6 +312,22 @@ pub fn set_open_file_limit(limit: usize) -> bool { false } +/// Simple helper to abstract away adding a forward-slash to a url if not present +/// +/// used mostly for deduplication purposes and url state tracking +pub fn normalize_url(url: &str) -> String { + log::trace!("enter: normalize_url({})", url); + + let normalized = if url.ends_with('/') { + url.to_string() + } else { + format!("{}/", url) + }; + + log::trace!("exit: normalize_url -> {}", normalized); + normalized +} + #[cfg(test)] mod tests { use super::*; From a2e13ea71ad7539ef439195fefb26e1a3ae8bf0e Mon Sep 17 00:00:00 2001 From: epi Date: Tue, 10 Nov 2020 07:16:31 -0600 Subject: [PATCH 02/18] added call to new scanner::initialize function --- src/main.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index 00d4464a..4e778953 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,7 +3,7 @@ use feroxbuster::{ banner, config::{CONFIGURATION, PROGRESS_BAR, PROGRESS_PRINTER}, heuristics, logger, reporter, - scanner::{scan_url, PAUSE_SCAN}, + scanner::{self, scan_url, PAUSE_SCAN}, utils::{ferox_print, get_current_depth, module_colorizer, status_colorizer}, FeroxError, FeroxResponse, FeroxResult, SLEEP_DURATION, VERSION, }; @@ -112,6 +112,8 @@ async fn scan( return Err(Box::new(err)); } + scanner::initialize(words.len(), CONFIGURATION.scan_limit, &CONFIGURATION.extensions, &CONFIGURATION.filter_status); + let mut tasks = vec![]; for target in targets { From d0a6c61de2ae1942a5c147a35af011688118a8b9 Mon Sep 17 00:00:00 2001 From: epi Date: Thu, 12 Nov 2020 06:54:09 -0600 Subject: [PATCH 03/18] pre master merge --- src/lib.rs | 183 +++++++++++++++++++++++++++++++++++++++- src/main.rs | 7 +- src/scanner.rs | 221 +++++++++++++++++++++++++++---------------------- 3 files changed, 308 insertions(+), 103 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 8051d86f..c6abf0fa 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,12 +11,19 @@ pub mod reporter; pub mod scanner; pub mod utils; +use indicatif::ProgressBar; use reqwest::{ header::HeaderMap, {Response, StatusCode, Url}, }; -use std::{error, fmt}; +use std::{ + cmp::PartialEq, + error, fmt, + sync::{Arc, Mutex}, +}; use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; +use tokio::task::JoinHandle; +use uuid::Uuid; /// Generic Result type to ease error handling in async contexts pub type FeroxResult = std::result::Result>; @@ -196,6 +203,180 @@ impl FeroxResponse { } } +/// Struct to hold scan-related state +/// +/// The purpose of this container is to open up the pathway to aborting currently running tasks and +/// serialization of all scan state into a state file in order to resume scans that were cut short +#[derive(Debug)] +struct FeroxScan { + /// UUID that uniquely ID's the scan + pub id: String, + + /// The URL that to be scanned + pub url: String, + + /// Whether or not this scan has completed + pub complete: bool, + + /// The spawned tokio task performing this scan + pub task: Option>, + + /// The progress bar associated with this scan + pub progress_bar: Option, +} + +/// Implementation of FeroxScan +impl FeroxScan { + /// Stop a currently running scan + pub fn abort(&self) { + if let Some(_task) = &self.task { + // task.abort(); todo uncomment once upgraded to tokio 0.3 + } + self.stop_progress_bar(); + } + + /// Create a default FeroxScan, populates ID with a new UUID + fn default() -> Self { + let new_id = Uuid::new_v4().to_simple().to_string(); + + FeroxScan { + id: new_id, + complete: false, + url: String::new(), + task: None, + progress_bar: None, + } + } + + /// Simple helper to call .finish on the scan's progress bar + fn stop_progress_bar(&self) { + if let Some(pb) = &self.progress_bar { + pb.finish(); + } + } + + /// Given a URL and ProgressBar, create a new FeroxScan, wrap it in an Arc and return it + pub fn new(url: &str, pb: ProgressBar) -> Arc> { + let mut me = Self::default(); + + me.url = utils::normalize_url(url); + me.progress_bar = Some(pb); + Arc::new(Mutex::new(me)) + } + + /// Mark the scan as complete and stop the scan's progress bar + pub fn finish(&mut self) { + self.complete = true; + self.stop_progress_bar(); + } +} + +// /// Eq implementation +// impl Eq for FeroxScan {} + +/// PartialEq implementation; uses FeroxScan.id for comparison +impl PartialEq for FeroxScan { + fn eq(&self, other: &Self) -> bool { + self.id == other.id + } +} + +// /// Hash implementation; uses uses FeroxScan.id and uses FeroxScan.url for hashing +// impl Hash for FeroxScan { +// /// Do the hashing with the hasher +// fn hash(&self, state: &mut H) { +// self.id.hash(state); +// self.url.hash(state); +// } +// } + +/// Container around a locked hashset of `FeroxScan`s, adds wrappers for insertion and searching +#[derive(Debug, Default)] +struct FeroxScans { + scans: Mutex>>>, +} + +/// Implementation of `FeroxScans` +impl FeroxScans { + /// Add a `FeroxScan` to the internal container + /// + /// If the internal container did NOT contain the scan, true is returned; else false + pub fn insert(&mut self, scan: Arc>) -> bool { + let sentry = match scan.lock() { + Ok(locked_scan) => { + // If the container did contain the scan, set sentry to false + // If the container did not contain the scan, set sentry to true + !self.contains(&locked_scan.url) + } + Err(e) => { + // poisoned lock + log::error!("FeroxScan's ({:?}) mutex is poisoned: {}", self, e); + false + } + }; + + if sentry { + // can't update the internal container while the scan itself is locked, so first + // lock the scan and check the container for the scan's presence, then add if + // not found + match self.scans.lock() { + Ok(mut scans) => { + scans.push(scan); + } + Err(e) => { + log::error!("FeroxScans' container's mutex is poisoned: {}", e); + return false; + } + } + } + + sentry + } + + /// Simple check for whether or not a FeroxScan is contained within the inner container based + /// on the given URL + pub fn contains(&self, url: &str) -> bool { + let normalized_url = utils::normalize_url(url); + + match self.scans.lock() { + Ok(scans) => { + for scan in scans.iter() { + if let Ok(locked_scan) = scan.lock() { + if locked_scan.url == normalized_url { + return true; + } + } + } + } + Err(e) => { + log::error!("FeroxScans' container's mutex is poisoned: {}", e); + } + } + false + } + + /// Find and return a `FeroxScan` based on the given URL + pub fn get_scan_by_url(&self, url: &str) -> Option>> { + let normalized_url = utils::normalize_url(url); + + match self.scans.lock() { + Ok(scans) => { + for scan in scans.iter() { + if let Ok(locked_scan) = scan.lock() { + if locked_scan.url == normalized_url { + return Some(scan.clone()); + } + } + } + } + Err(e) => { + log::error!("FeroxScans' container's mutex is poisoned: {}", e); + } + } + None + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/main.rs b/src/main.rs index 4e778953..42419bb6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -112,7 +112,12 @@ async fn scan( return Err(Box::new(err)); } - scanner::initialize(words.len(), CONFIGURATION.scan_limit, &CONFIGURATION.extensions, &CONFIGURATION.filter_status); + scanner::initialize( + words.len(), + CONFIGURATION.scan_limit, + &CONFIGURATION.extensions, + &CONFIGURATION.filter_status, + ); let mut tasks = vec![]; diff --git a/src/scanner.rs b/src/scanner.rs index f7832515..3b751281 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -1,26 +1,25 @@ use crate::{ - config::CONFIGURATION, + config::{CONFIGURATION, PROGRESS_PRINTER}, extractor::get_links, filters::{FeroxFilter, StatusCodeFilter, WildcardFilter}, heuristics, progress, utils::{format_url, get_current_depth, make_request}, - FeroxChannel, FeroxResponse, SLEEP_DURATION, + FeroxChannel, FeroxResponse, FeroxScan, FeroxScans, SLEEP_DURATION, }; -use console::style; use futures::{ future::{BoxFuture, FutureExt}, stream, StreamExt, }; -use indicatif::{ProgressBar, ProgressStyle}; use lazy_static::lazy_static; use reqwest::Url; +use std::sync::atomic::AtomicU64; use std::{ collections::HashSet, convert::TryInto, io::{stderr, Write}, ops::Deref, sync::atomic::{AtomicBool, AtomicUsize, Ordering}, - sync::{Arc, RwLock}, + sync::{Arc, Mutex, RwLock}, }; use tokio::{ sync::{ @@ -34,15 +33,22 @@ use tokio::{ /// Single atomic number that gets incremented once, used to track first scan vs. all others static CALL_COUNT: AtomicUsize = AtomicUsize::new(0); +/// Single atomic number that gets holds the number of requests to be sent per directory scanned +static NUMBER_OF_REQUESTS: AtomicU64 = AtomicU64::new(0); + +/// Single atomic number that gets incremented once, used to track first thread to interact with +/// when pausing a scan +static INTERACTIVE_BARRIER: AtomicUsize = AtomicUsize::new(0); + /// Atomic boolean flag, used to determine whether or not a scan should pause or resume pub static PAUSE_SCAN: AtomicBool = AtomicBool::new(false); lazy_static! { /// Set of urls that have been sent to [scan_url](fn.scan_url.html), used for deduplication - static ref SCANNED_URLS: RwLock> = RwLock::new(HashSet::new()); + static ref SCANNED_URLS: FeroxScans = FeroxScans::default(); - /// A clock spinner protected with a RwLock to allow for a single thread to use at a time - static ref SINGLE_SPINNER: RwLock = RwLock::new(get_single_spinner()); + // /// A clock spinner protected with a RwLock to allow for a single thread to use at a time + // static ref BARRIER: Arc> = Arc::new(RwLock::new(true)); /// Vector of implementors of the FeroxFilter trait static ref FILTERS: Arc>>> = Arc::new(RwLock::new(Vec::>::new())); @@ -51,25 +57,6 @@ lazy_static! { static ref SCAN_LIMITER: Semaphore = Semaphore::new(CONFIGURATION.scan_limit); } -/// Return a clock spinner, used when scans are paused -fn get_single_spinner() -> ProgressBar { - log::trace!("enter: get_single_spinner"); - - let spinner = ProgressBar::new_spinner().with_style( - ProgressStyle::default_spinner() - .tick_strings(&[ - "🕛", "🕐", "🕑", "🕒", "🕓", "🕔", "🕕", "🕖", "🕗", "🕘", "🕙", "🕚", - ]) - .template(&format!( - "\t-= All Scans {{spinner}} {} =-", - style("Paused").red() - )), - ); - - log::trace!("exit: get_single_spinner -> {:?}", spinner); - spinner -} - /// Forced the calling thread into a busy loop /// /// Every `SLEEP_DURATION` milliseconds, the function examines the result stored in `PAUSE_SCAN` @@ -87,18 +74,10 @@ async fn pause_scan() { // ignore any error returned let _ = stderr().flush(); - if SINGLE_SPINNER.read().unwrap().is_finished() { - // in order to not leave draw artifacts laying around in the terminal, we call - // finish_and_clear on the progress bar when resuming scans. For this reason, we need to - // check if the spinner is finished, and repopulate the RwLock with a new spinner if - // necessary - if let Ok(mut guard) = SINGLE_SPINNER.write() { - *guard = get_single_spinner(); - } - } + if INTERACTIVE_BARRIER.load(Ordering::Relaxed) == 0 { + INTERACTIVE_BARRIER.fetch_add(1, Ordering::Relaxed); - if let Ok(spinner) = SINGLE_SPINNER.write() { - spinner.enable_steady_tick(120); + PROGRESS_PRINTER.println(format!("Here's your shit: {:?}", SCANNED_URLS.scans)); } loop { @@ -107,51 +86,46 @@ async fn pause_scan() { if !PAUSE_SCAN.load(Ordering::Acquire) { // PAUSE_SCAN is false, so we can exit the busy loop - if let Ok(spinner) = SINGLE_SPINNER.write() { - spinner.finish_and_clear(); - } let _ = stderr().flush(); + if INTERACTIVE_BARRIER.load(Ordering::Relaxed) == 1 { + INTERACTIVE_BARRIER.fetch_sub(1, Ordering::Relaxed); + } log::trace!("exit: pause_scan"); return; } } } -/// Adds the given url to `SCANNED_URLS` +/// Given a url, create a new `FeroxScan` and add it to `FeroxScans` +/// +/// If `FeroxScans` did not already contain the scan, return true; otherwise return false /// -/// If `SCANNED_URLS` did not already contain the url, return true; otherwise return false -fn add_url_to_list_of_scanned_urls(resp: &str, scanned_urls: &RwLock>) -> bool { +/// Also return a reference to the new `FeroxScan` +fn add_url_to_list_of_scanned_urls( + url: &str, + scanned_urls: &mut FeroxScans, +) -> (bool, Arc>) { log::trace!( "enter: add_url_to_list_of_scanned_urls({}, {:?})", - resp, + url, scanned_urls ); - match scanned_urls.write() { - // check new url against what's already been scanned - Ok(mut urls) => { - let normalized_url = if resp.ends_with('/') { - // append a / to the list of 'seen' urls, this is to prevent the case where - // 3xx and 2xx duplicate eachother - resp.to_string() - } else { - format!("{}/", resp) - }; - - // If the set did not contain resp, true is returned. - // If the set did contain resp, false is returned. - let response = urls.insert(normalized_url); - - log::trace!("exit: add_url_to_list_of_scanned_urls -> {}", response); - response - } - Err(e) => { - // poisoned lock - log::error!("Set of scanned urls poisoned: {}", e); - log::trace!("exit: add_url_to_list_of_scanned_urls -> false"); - false - } - } + let progress_bar = progress::add_bar(&url, NUMBER_OF_REQUESTS.load(Ordering::Relaxed), false); + progress_bar.reset_elapsed(); + + let ferox_scan = FeroxScan::new(&url, progress_bar); + + // If the set did not contain the scan, true is returned. + // If the set did contain the scan, false is returned. + let response = scanned_urls.insert(ferox_scan.clone()); + + log::trace!( + "exit: add_url_to_list_of_scanned_urls -> ({}, {:?})", + response, + ferox_scan + ); + (response, ferox_scan) } /// Adds the given FeroxFilter to the given list of FeroxFilter implementors @@ -213,7 +187,7 @@ fn spawn_recursion_handler( let mut scans = vec![]; while let Some(resp) = recursion_channel.recv().await { - let unknown = add_url_to_list_of_scanned_urls(&resp, &SCANNED_URLS); + let (unknown, _ferox_scan) = add_url_to_list_of_scanned_urls(&resp, &SCANNED_URLS); if !unknown { // not unknown, i.e. we've seen the url before and don't need to scan again @@ -227,7 +201,7 @@ fn spawn_recursion_handler( let resp_clone = resp.clone(); let list_clone = wordlist.clone(); - scans.push(tokio::spawn(async move { + let future = tokio::spawn(async move { scan_url( resp_clone.to_owned().as_str(), list_clone, @@ -236,7 +210,9 @@ fn spawn_recursion_handler( file_clone, ) .await - })); + }); + + scans.push(future); } scans } @@ -496,7 +472,7 @@ async fn make_requests( let new_links = get_links(&ferox_response).await; for new_link in new_links { - let unknown = add_url_to_list_of_scanned_urls(&new_link, &SCANNED_URLS); + let (unknown, _) = add_url_to_list_of_scanned_urls(&new_link, &SCANNED_URLS); if !unknown { // not unknown, i.e. we've seen the url before and don't need to scan again @@ -608,31 +584,37 @@ pub async fn scan_url( let (tx_dir, rx_dir): FeroxChannel = mpsc::unbounded_channel(); - let num_reqs_expected: u64 = if CONFIGURATION.extensions.is_empty() { - wordlist.len().try_into().unwrap() - } else { - let total = wordlist.len() * (CONFIGURATION.extensions.len() + 1); - total.try_into().unwrap() - }; - - let progress_bar = progress::add_bar(&target_url, num_reqs_expected, false); - progress_bar.reset_elapsed(); - if CALL_COUNT.load(Ordering::Relaxed) == 0 { CALL_COUNT.fetch_add(1, Ordering::Relaxed); // this protection allows us to add the first scanned url to SCANNED_URLS // from within the scan_url function instead of the recursion handler add_url_to_list_of_scanned_urls(&target_url, &SCANNED_URLS); + } - if CONFIGURATION.scan_limit == 0 { - // scan_limit == 0 means no limit should be imposed... however, scoping the Semaphore - // permit is tricky, so as a workaround, we'll add a ridiculous number of permits to - // the semaphore (1,152,921,504,606,846,975 to be exact) and call that 'unlimited' - SCAN_LIMITER.add_permits(usize::MAX >> 4); - } + let ferox_scan = SCANNED_URLS.get_scan_by_url(&target_url); + + if ferox_scan.is_none() { + // todo probably remove this, fine for testing for now + log::error!( + "Could not find FeroxScan associated with {}; exiting scan", + target_url + ); + return; } + let ferox_scan = ferox_scan.unwrap(); + + // todo unwrap + let progress_bar = ferox_scan + .lock() + .unwrap() + .progress_bar + .as_ref() + .unwrap() + .clone(); + progress_bar.reset_elapsed(); + // When acquire is called and the semaphore has remaining permits, the function immediately // returns a permit. However, if no remaining permits are available, acquire (asynchronously) // waits until an outstanding permit is dropped. At this point, the freed permit is assigned @@ -667,15 +649,6 @@ pub async fn scan_url( add_filter_to_list_of_ferox_filters(filter, FILTERS.clone()); - // add any status code filters to `FILTERS` - for code_filter in &CONFIGURATION.filter_status { - let filter = StatusCodeFilter { - filter_code: *code_filter, - }; - let boxed_filter = Box::new(filter); - add_filter_to_list_of_ferox_filters(boxed_filter, FILTERS.clone()); - } - // producer tasks (mp of mpsc); responsible for making requests let producers = stream::iter(looping_words.deref().to_owned()) .map(|word| { @@ -715,7 +688,8 @@ pub async fn scan_url( // drop the current permit so the semaphore will allow another scan to proceed drop(permit); - progress_bar.finish(); + // todo unwrap + ferox_scan.lock().unwrap().finish(); // manually drop tx in order for the rx task's while loops to eval to false log::trace!("dropped recursion handler's transmitter"); @@ -729,6 +703,51 @@ pub async fn scan_url( log::trace!("exit: scan_url"); } +/// Perform steps necessary to run scans that only need to be performed once (warming up the +/// engine, as it were) +pub fn initialize( + num_words: usize, + scan_limit: usize, + extensions: &[String], + status_code_filters: &[u16], +) { + log::trace!( + "enter: initialize({}, {}, {:?}, {:?})", + num_words, + scan_limit, + extensions, + status_code_filters + ); + + // number of requests only needs to be calculated once, and then can be reused + let num_reqs_expected: u64 = if extensions.is_empty() { + num_words.try_into().unwrap() + } else { + let total = num_words * (extensions.len() + 1); + total.try_into().unwrap() + }; + + NUMBER_OF_REQUESTS.store(num_reqs_expected, Ordering::Relaxed); + + // add any status code filters to `FILTERS` + for code_filter in status_code_filters { + let filter = StatusCodeFilter { + filter_code: *code_filter, + }; + let boxed_filter = Box::new(filter); + add_filter_to_list_of_ferox_filters(boxed_filter, FILTERS.clone()); + } + + if scan_limit == 0 { + // scan_limit == 0 means no limit should be imposed... however, scoping the Semaphore + // permit is tricky, so as a workaround, we'll add a ridiculous number of permits to + // the semaphore (1,152,921,504,606,846,975 to be exact) and call that 'unlimited' + SCAN_LIMITER.add_permits(usize::MAX >> 4); + } + + log::trace!("exit: initialize"); +} + #[cfg(test)] mod tests { use super::*; @@ -877,7 +896,7 @@ mod tests { let now = time::Instant::now(); PAUSE_SCAN.store(true, Ordering::Relaxed); - SINGLE_SPINNER.write().unwrap().finish_and_clear(); + // BARRIER.write().unwrap().finish_and_clear(); let expected = time::Duration::from_secs(2); From b00a47e5e50e9063a69116c5479837d255ec3776 Mon Sep 17 00:00:00 2001 From: epi Date: Thu, 12 Nov 2020 15:00:49 -0600 Subject: [PATCH 04/18] moved functions related to scan management into their own module --- src/lib.rs | 189 +-------------------------- src/main.rs | 3 +- src/scan_manager.rs | 312 ++++++++++++++++++++++++++++++++++++++++++++ src/scanner.rs | 168 ++---------------------- src/utils.rs | 8 +- 5 files changed, 331 insertions(+), 349 deletions(-) create mode 100644 src/scan_manager.rs diff --git a/src/lib.rs b/src/lib.rs index c6abf0fa..85fc96d9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,22 +8,13 @@ pub mod logger; pub mod parser; pub mod progress; pub mod reporter; +pub mod scan_manager; pub mod scanner; pub mod utils; -use indicatif::ProgressBar; -use reqwest::{ - header::HeaderMap, - {Response, StatusCode, Url}, -}; -use std::{ - cmp::PartialEq, - error, fmt, - sync::{Arc, Mutex}, -}; +use reqwest::{header::HeaderMap, Response, StatusCode, Url}; +use std::{error, fmt}; use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; -use tokio::task::JoinHandle; -use uuid::Uuid; /// Generic Result type to ease error handling in async contexts pub type FeroxResult = std::result::Result>; @@ -203,180 +194,6 @@ impl FeroxResponse { } } -/// Struct to hold scan-related state -/// -/// The purpose of this container is to open up the pathway to aborting currently running tasks and -/// serialization of all scan state into a state file in order to resume scans that were cut short -#[derive(Debug)] -struct FeroxScan { - /// UUID that uniquely ID's the scan - pub id: String, - - /// The URL that to be scanned - pub url: String, - - /// Whether or not this scan has completed - pub complete: bool, - - /// The spawned tokio task performing this scan - pub task: Option>, - - /// The progress bar associated with this scan - pub progress_bar: Option, -} - -/// Implementation of FeroxScan -impl FeroxScan { - /// Stop a currently running scan - pub fn abort(&self) { - if let Some(_task) = &self.task { - // task.abort(); todo uncomment once upgraded to tokio 0.3 - } - self.stop_progress_bar(); - } - - /// Create a default FeroxScan, populates ID with a new UUID - fn default() -> Self { - let new_id = Uuid::new_v4().to_simple().to_string(); - - FeroxScan { - id: new_id, - complete: false, - url: String::new(), - task: None, - progress_bar: None, - } - } - - /// Simple helper to call .finish on the scan's progress bar - fn stop_progress_bar(&self) { - if let Some(pb) = &self.progress_bar { - pb.finish(); - } - } - - /// Given a URL and ProgressBar, create a new FeroxScan, wrap it in an Arc and return it - pub fn new(url: &str, pb: ProgressBar) -> Arc> { - let mut me = Self::default(); - - me.url = utils::normalize_url(url); - me.progress_bar = Some(pb); - Arc::new(Mutex::new(me)) - } - - /// Mark the scan as complete and stop the scan's progress bar - pub fn finish(&mut self) { - self.complete = true; - self.stop_progress_bar(); - } -} - -// /// Eq implementation -// impl Eq for FeroxScan {} - -/// PartialEq implementation; uses FeroxScan.id for comparison -impl PartialEq for FeroxScan { - fn eq(&self, other: &Self) -> bool { - self.id == other.id - } -} - -// /// Hash implementation; uses uses FeroxScan.id and uses FeroxScan.url for hashing -// impl Hash for FeroxScan { -// /// Do the hashing with the hasher -// fn hash(&self, state: &mut H) { -// self.id.hash(state); -// self.url.hash(state); -// } -// } - -/// Container around a locked hashset of `FeroxScan`s, adds wrappers for insertion and searching -#[derive(Debug, Default)] -struct FeroxScans { - scans: Mutex>>>, -} - -/// Implementation of `FeroxScans` -impl FeroxScans { - /// Add a `FeroxScan` to the internal container - /// - /// If the internal container did NOT contain the scan, true is returned; else false - pub fn insert(&mut self, scan: Arc>) -> bool { - let sentry = match scan.lock() { - Ok(locked_scan) => { - // If the container did contain the scan, set sentry to false - // If the container did not contain the scan, set sentry to true - !self.contains(&locked_scan.url) - } - Err(e) => { - // poisoned lock - log::error!("FeroxScan's ({:?}) mutex is poisoned: {}", self, e); - false - } - }; - - if sentry { - // can't update the internal container while the scan itself is locked, so first - // lock the scan and check the container for the scan's presence, then add if - // not found - match self.scans.lock() { - Ok(mut scans) => { - scans.push(scan); - } - Err(e) => { - log::error!("FeroxScans' container's mutex is poisoned: {}", e); - return false; - } - } - } - - sentry - } - - /// Simple check for whether or not a FeroxScan is contained within the inner container based - /// on the given URL - pub fn contains(&self, url: &str) -> bool { - let normalized_url = utils::normalize_url(url); - - match self.scans.lock() { - Ok(scans) => { - for scan in scans.iter() { - if let Ok(locked_scan) = scan.lock() { - if locked_scan.url == normalized_url { - return true; - } - } - } - } - Err(e) => { - log::error!("FeroxScans' container's mutex is poisoned: {}", e); - } - } - false - } - - /// Find and return a `FeroxScan` based on the given URL - pub fn get_scan_by_url(&self, url: &str) -> Option>> { - let normalized_url = utils::normalize_url(url); - - match self.scans.lock() { - Ok(scans) => { - for scan in scans.iter() { - if let Ok(locked_scan) = scan.lock() { - if locked_scan.url == normalized_url { - return Some(scan.clone()); - } - } - } - } - Err(e) => { - log::error!("FeroxScans' container's mutex is poisoned: {}", e); - } - } - None - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/src/main.rs b/src/main.rs index 42419bb6..6942f874 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,7 +3,8 @@ use feroxbuster::{ banner, config::{CONFIGURATION, PROGRESS_BAR, PROGRESS_PRINTER}, heuristics, logger, reporter, - scanner::{self, scan_url, PAUSE_SCAN}, + scanner::{self, scan_url}, + scan_manager::PAUSE_SCAN, utils::{ferox_print, get_current_depth, module_colorizer, status_colorizer}, FeroxError, FeroxResponse, FeroxResult, SLEEP_DURATION, VERSION, }; diff --git a/src/scan_manager.rs b/src/scan_manager.rs new file mode 100644 index 00000000..2c79a7c8 --- /dev/null +++ b/src/scan_manager.rs @@ -0,0 +1,312 @@ +use crate::{config::PROGRESS_PRINTER, progress, SLEEP_DURATION, scanner::NUMBER_OF_REQUESTS}; +use indicatif::ProgressBar; +use std::{ + cmp::PartialEq, + sync::{Arc, Mutex}, +}; +use std::{ + io::{stderr, Write}, + sync::atomic::{AtomicBool, AtomicUsize, Ordering}, +}; +use tokio::{task::JoinHandle, time}; +use uuid::Uuid; + +/// Single atomic number that gets incremented once, used to track first thread to interact with +/// when pausing a scan +static INTERACTIVE_BARRIER: AtomicUsize = AtomicUsize::new(0); + +/// Atomic boolean flag, used to determine whether or not a scan should pause or resume +pub static PAUSE_SCAN: AtomicBool = AtomicBool::new(false); + +/// Struct to hold scan-related state +/// +/// The purpose of this container is to open up the pathway to aborting currently running tasks and +/// serialization of all scan state into a state file in order to resume scans that were cut short +#[derive(Debug)] +pub struct FeroxScan { + /// UUID that uniquely ID's the scan + pub id: String, + + /// The URL that to be scanned + pub url: String, + + /// Whether or not this scan has completed + pub complete: bool, + + /// The spawned tokio task performing this scan + pub task: Option>, + + /// The progress bar associated with this scan + pub progress_bar: Option, +} + +/// Implementation of FeroxScan +impl FeroxScan { + /// Stop a currently running scan + pub fn abort(&self) { + if let Some(_task) = &self.task { + // task.abort(); todo uncomment once upgraded to tokio 0.3 + } + self.stop_progress_bar(); + } + + /// Create a default FeroxScan, populates ID with a new UUID + fn default() -> Self { + let new_id = Uuid::new_v4().to_simple().to_string(); + + FeroxScan { + id: new_id, + complete: false, + url: String::new(), + task: None, + progress_bar: None, + } + } + + /// Simple helper to call .finish on the scan's progress bar + fn stop_progress_bar(&self) { + if let Some(pb) = &self.progress_bar { + pb.finish(); + } + } + + /// Given a URL and ProgressBar, create a new FeroxScan, wrap it in an Arc and return it + pub fn new(url: &str, pb: ProgressBar) -> Arc> { + let mut me = Self::default(); + + me.url = url.to_string(); + me.progress_bar = Some(pb); + Arc::new(Mutex::new(me)) + } + + /// Mark the scan as complete and stop the scan's progress bar + pub fn finish(&mut self) { + self.complete = true; + self.stop_progress_bar(); + } +} + +// /// Eq implementation +// impl Eq for FeroxScan {} + +/// PartialEq implementation; uses FeroxScan.id for comparison +impl PartialEq for FeroxScan { + fn eq(&self, other: &Self) -> bool { + self.id == other.id + } +} + +/// Container around a locked hashset of `FeroxScan`s, adds wrappers for insertion and searching +#[derive(Debug, Default)] +pub struct FeroxScans { + /// Internal structure: locked hashset of `FeroxScan`s + pub scans: Mutex>>>, +} + +/// Implementation of `FeroxScans` +impl FeroxScans { + /// Add a `FeroxScan` to the internal container + /// + /// If the internal container did NOT contain the scan, true is returned; else false + pub fn insert(&self, scan: Arc>) -> bool { + let sentry = match scan.lock() { + Ok(locked_scan) => { + // If the container did contain the scan, set sentry to false + // If the container did not contain the scan, set sentry to true + !self.contains(&locked_scan.url) + } + Err(e) => { + // poisoned lock + log::error!("FeroxScan's ({:?}) mutex is poisoned: {}", self, e); + false + } + }; + + if sentry { + // can't update the internal container while the scan itself is locked, so first + // lock the scan and check the container for the scan's presence, then add if + // not found + match self.scans.lock() { + Ok(mut scans) => { + scans.push(scan); + } + Err(e) => { + log::error!("FeroxScans' container's mutex is poisoned: {}", e); + return false; + } + } + } + + sentry + } + + /// Simple check for whether or not a FeroxScan is contained within the inner container based + /// on the given URL + pub fn contains(&self, url: &str) -> bool { + match self.scans.lock() { + Ok(scans) => { + for scan in scans.iter() { + if let Ok(locked_scan) = scan.lock() { + if locked_scan.url == url { + return true; + } + } + } + } + Err(e) => { + log::error!("FeroxScans' container's mutex is poisoned: {}", e); + } + } + false + } + + /// Find and return a `FeroxScan` based on the given URL + pub fn get_scan_by_url(&self, url: &str) -> Option>> { + match self.scans.lock() { + Ok(scans) => { + for scan in scans.iter() { + if let Ok(locked_scan) = scan.lock() { + if locked_scan.url == url { + return Some(scan.clone()); + } + } + } + } + Err(e) => { + log::error!("FeroxScans' container's mutex is poisoned: {}", e); + } + } + None + } + + /// Forced the calling thread into a busy loop + /// + /// Every `SLEEP_DURATION` milliseconds, the function examines the result stored in `PAUSE_SCAN` + /// + /// When the value stored in `PAUSE_SCAN` becomes `false`, the function returns, exiting the busy + /// loop + pub async fn pause(&self) { + log::trace!("enter: pause_scan"); + // function uses tokio::time, not std + + // local testing showed a pretty slow increase (less than linear) in CPU usage as # of + // concurrent scans rose when SLEEP_DURATION was set to 500, using that as the default for now + let mut interval = time::interval(time::Duration::from_millis(SLEEP_DURATION)); + + // ignore any error returned + let _ = stderr().flush(); + + if INTERACTIVE_BARRIER.load(Ordering::Relaxed) == 0 { + INTERACTIVE_BARRIER.fetch_add(1, Ordering::Relaxed); + + PROGRESS_PRINTER.println(format!("Here's your shit: {:?}", self.scans)); + let mut s = String::new(); + std::io::stdin().read_line(&mut s).unwrap(); + PROGRESS_PRINTER.println(format!("Here's your shit: {}", s)); + } + + loop { + // first tick happens immediately, all others wait the specified duration + interval.tick().await; + + if !PAUSE_SCAN.load(Ordering::Acquire) { + // PAUSE_SCAN is false, so we can exit the busy loop + let _ = stderr().flush(); + if INTERACTIVE_BARRIER.load(Ordering::Relaxed) == 1 { + INTERACTIVE_BARRIER.fetch_sub(1, Ordering::Relaxed); + } + log::trace!("exit: pause_scan"); + return; + } + } + } + + /// Given a url, create a new `FeroxScan` and add it to `FeroxScans` + /// + /// If `FeroxScans` did not already contain the scan, return true; otherwise return false + /// + /// Also return a reference to the new `FeroxScan` + pub fn add_scan(&self, url: &str) -> (bool, Arc>) { + let progress_bar = + progress::add_bar(&url, NUMBER_OF_REQUESTS.load(Ordering::Relaxed), false); + + progress_bar.reset_elapsed(); + + let ferox_scan = FeroxScan::new(&url, progress_bar); + + // If the set did not contain the scan, true is returned. + // If the set did contain the scan, false is returned. + let response = self.insert(ferox_scan.clone()); + + (response, ferox_scan) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // todo add_url_* and pause_scan tests need to be redone + + #[tokio::test(core_threads = 1)] + /// tests that pause_scan pauses execution and releases execution when PAUSE_SCAN is toggled + /// the spinner used during the test has had .finish_and_clear called on it, meaning that + /// a new one will be created, taking the if branch within the function + async fn scanner_pause_scan_with_finished_spinner() { + let now = time::Instant::now(); + + PAUSE_SCAN.store(true, Ordering::Relaxed); + // BARRIER.write().unwrap().finish_and_clear(); + + let expected = time::Duration::from_secs(2); + + tokio::spawn(async move { + time::delay_for(expected).await; + PAUSE_SCAN.store(false, Ordering::Relaxed); + }); + + pause_scan().await; + + assert!(now.elapsed() > expected); + } + + #[test] + /// add an unknown url to the hashset, expect true + fn add_url_to_list_of_scanned_urls_with_unknown_url() { + let urls = FeroxScans::default(); + let url = "http://unknown_url"; + let (result, _scan) = add_url_to_list_of_scanned_urls(url, &urls); + assert_eq!(result, true); + } + + #[test] + /// add a known url to the hashset, with a trailing slash, expect false + fn add_url_to_list_of_scanned_urls_with_known_url() { + let urls = FeroxScans::default(); + let pb = ProgressBar::new(1); + let url = "http://unknown_url/"; + let mut scan = FeroxScan::new(url, pb); + + assert_eq!(urls.insert(scan), true); + + let (result, _scan) = add_url_to_list_of_scanned_urls(url, &urls); + + assert_eq!(result, false); + } + + #[test] + /// add a known url to the hashset, without a trailing slash, expect false + fn add_url_to_list_of_scanned_urls_with_known_url_without_slash() { + let urls = FeroxScans::default(); + let pb = ProgressBar::new(1); + let url = "http://unknown_url"; + let mut scan = FeroxScan::new(url, pb); + + assert_eq!(urls.insert(scan), true); + + let (result, _scan) = add_url_to_list_of_scanned_urls(url, &urls); + + assert_eq!(result, false); + } + +} \ No newline at end of file diff --git a/src/scanner.rs b/src/scanner.rs index aaaafac2..bbfd331b 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -1,10 +1,11 @@ use crate::{ - config::{CONFIGURATION, PROGRESS_PRINTER}, + config::CONFIGURATION, extractor::get_links, filters::{FeroxFilter, StatusCodeFilter, WildcardFilter}, - heuristics, progress, + heuristics, + scan_manager::{FeroxScans, PAUSE_SCAN}, utils::{format_url, get_current_depth, make_request}, - FeroxChannel, FeroxResponse, FeroxScan, FeroxScans, SLEEP_DURATION, + FeroxChannel, FeroxResponse, }; use futures::{ future::{BoxFuture, FutureExt}, @@ -12,14 +13,12 @@ use futures::{ }; use lazy_static::lazy_static; use reqwest::Url; -use std::sync::atomic::AtomicU64; use std::{ collections::HashSet, convert::TryInto, - io::{stderr, Write}, ops::Deref, - sync::atomic::{AtomicBool, AtomicUsize, Ordering}, - sync::{Arc, Mutex, RwLock}, + sync::atomic::{AtomicU64, AtomicUsize, Ordering}, + sync::{Arc, RwLock}, }; use tokio::{ sync::{ @@ -27,26 +26,19 @@ use tokio::{ Semaphore, }, task::JoinHandle, - time, }; /// Single atomic number that gets incremented once, used to track first scan vs. all others static CALL_COUNT: AtomicUsize = AtomicUsize::new(0); /// Single atomic number that gets holds the number of requests to be sent per directory scanned -static NUMBER_OF_REQUESTS: AtomicU64 = AtomicU64::new(0); - -/// Single atomic number that gets incremented once, used to track first thread to interact with -/// when pausing a scan -static INTERACTIVE_BARRIER: AtomicUsize = AtomicUsize::new(0); - -/// Atomic boolean flag, used to determine whether or not a scan should pause or resume -pub static PAUSE_SCAN: AtomicBool = AtomicBool::new(false); +pub static NUMBER_OF_REQUESTS: AtomicU64 = AtomicU64::new(0); lazy_static! { /// Set of urls that have been sent to [scan_url](fn.scan_url.html), used for deduplication static ref SCANNED_URLS: FeroxScans = FeroxScans::default(); + // todo remove if not needed // /// A clock spinner protected with a RwLock to allow for a single thread to use at a time // static ref BARRIER: Arc> = Arc::new(RwLock::new(true)); @@ -57,77 +49,6 @@ lazy_static! { static ref SCAN_LIMITER: Semaphore = Semaphore::new(CONFIGURATION.scan_limit); } -/// Forced the calling thread into a busy loop -/// -/// Every `SLEEP_DURATION` milliseconds, the function examines the result stored in `PAUSE_SCAN` -/// -/// When the value stored in `PAUSE_SCAN` becomes `false`, the function returns, exiting the busy -/// loop -async fn pause_scan() { - log::trace!("enter: pause_scan"); - // function uses tokio::time, not std - - // local testing showed a pretty slow increase (less than linear) in CPU usage as # of - // concurrent scans rose when SLEEP_DURATION was set to 500, using that as the default for now - let mut interval = time::interval(time::Duration::from_millis(SLEEP_DURATION)); - - // ignore any error returned - let _ = stderr().flush(); - - if INTERACTIVE_BARRIER.load(Ordering::Relaxed) == 0 { - INTERACTIVE_BARRIER.fetch_add(1, Ordering::Relaxed); - - PROGRESS_PRINTER.println(format!("Here's your shit: {:?}", SCANNED_URLS.scans)); - } - - loop { - // first tick happens immediately, all others wait the specified duration - interval.tick().await; - - if !PAUSE_SCAN.load(Ordering::Acquire) { - // PAUSE_SCAN is false, so we can exit the busy loop - let _ = stderr().flush(); - if INTERACTIVE_BARRIER.load(Ordering::Relaxed) == 1 { - INTERACTIVE_BARRIER.fetch_sub(1, Ordering::Relaxed); - } - log::trace!("exit: pause_scan"); - return; - } - } -} - -/// Given a url, create a new `FeroxScan` and add it to `FeroxScans` -/// -/// If `FeroxScans` did not already contain the scan, return true; otherwise return false -/// -/// Also return a reference to the new `FeroxScan` -fn add_url_to_list_of_scanned_urls( - url: &str, - scanned_urls: &mut FeroxScans, -) -> (bool, Arc>) { - log::trace!( - "enter: add_url_to_list_of_scanned_urls({}, {:?})", - url, - scanned_urls - ); - - let progress_bar = progress::add_bar(&url, NUMBER_OF_REQUESTS.load(Ordering::Relaxed), false); - progress_bar.reset_elapsed(); - - let ferox_scan = FeroxScan::new(&url, progress_bar); - - // If the set did not contain the scan, true is returned. - // If the set did contain the scan, false is returned. - let response = scanned_urls.insert(ferox_scan.clone()); - - log::trace!( - "exit: add_url_to_list_of_scanned_urls -> ({}, {:?})", - response, - ferox_scan - ); - (response, ferox_scan) -} - /// Adds the given FeroxFilter to the given list of FeroxFilter implementors /// /// If the given list did not already contain the filter, return true; otherwise return false @@ -187,7 +108,7 @@ fn spawn_recursion_handler( let mut scans = vec![]; while let Some(resp) = recursion_channel.recv().await { - let (unknown, _ferox_scan) = add_url_to_list_of_scanned_urls(&resp, &SCANNED_URLS); + let (unknown, _ferox_scan) = SCANNED_URLS.add_scan(&resp); if !unknown { // not unknown, i.e. we've seen the url before and don't need to scan again @@ -472,7 +393,7 @@ async fn make_requests( let new_links = get_links(&ferox_response).await; for new_link in new_links { - let (unknown, _) = add_url_to_list_of_scanned_urls(&new_link, &SCANNED_URLS); + let (unknown, _) = SCANNED_URLS.add_scan(&new_link); if !unknown { // not unknown, i.e. we've seen the url before and don't need to scan again @@ -589,7 +510,7 @@ pub async fn scan_url( // this protection allows us to add the first scanned url to SCANNED_URLS // from within the scan_url function instead of the recursion handler - add_url_to_list_of_scanned_urls(&target_url, &SCANNED_URLS); + SCANNED_URLS.add_scan(&target_url); } let ferox_scan = SCANNED_URLS.get_scan_by_url(&target_url); @@ -662,7 +583,7 @@ pub async fn scan_url( // for every word in the wordlist, check to see if PAUSE_SCAN is set to true // when true; enter a busy loop that only exits by setting PAUSE_SCAN back // to false - pause_scan().await; + SCANNED_URLS.pause().await; } make_requests(&tgt, &word, base_depth, txd, txr).await }), @@ -751,6 +672,7 @@ pub fn initialize( #[cfg(test)] mod tests { use super::*; + use indicatif::ProgressBar; #[test] /// sending url + word without any extensions should get back one url with the joined word @@ -845,68 +767,4 @@ mod tests { let result = reached_max_depth(&url, 0, 2); assert!(result); } - - #[test] - /// add an unknown url to the hashset, expect true - fn add_url_to_list_of_scanned_urls_with_unknown_url() { - let urls = RwLock::new(HashSet::::new()); - let url = "http://unknown_url"; - assert_eq!(add_url_to_list_of_scanned_urls(url, &urls), true); - } - - #[test] - /// add a known url to the hashset, with a trailing slash, expect false - fn add_url_to_list_of_scanned_urls_with_known_url() { - let urls = RwLock::new(HashSet::::new()); - let url = "http://unknown_url/"; - - assert_eq!(urls.write().unwrap().insert(url.to_string()), true); - - assert_eq!(add_url_to_list_of_scanned_urls(url, &urls), false); - } - - #[test] - /// add a known url to the hashset, without a trailing slash, expect false - fn add_url_to_list_of_scanned_urls_with_known_url_without_slash() { - let urls = RwLock::new(HashSet::::new()); - let url = "http://unknown_url"; - - assert_eq!( - urls.write() - .unwrap() - .insert("http://unknown_url".to_string()), - true - ); - - assert_eq!(add_url_to_list_of_scanned_urls(url, &urls), false); - } - - #[test] - /// test that get_single_spinner returns the correct spinner - fn scanner_get_single_spinner_returns_spinner() { - let spinner = get_single_spinner(); - assert!(!spinner.is_finished()); - } - - #[tokio::test(core_threads = 1)] - /// tests that pause_scan pauses execution and releases execution when PAUSE_SCAN is toggled - /// the spinner used during the test has had .finish_and_clear called on it, meaning that - /// a new one will be created, taking the if branch within the function - async fn scanner_pause_scan_with_finished_spinner() { - let now = time::Instant::now(); - - PAUSE_SCAN.store(true, Ordering::Relaxed); - // BARRIER.write().unwrap().finish_and_clear(); - - let expected = time::Duration::from_secs(2); - - tokio::spawn(async move { - time::delay_for(expected).await; - PAUSE_SCAN.store(false, Ordering::Relaxed); - }); - - pause_scan().await; - - assert!(now.elapsed() > expected); - } } diff --git a/src/utils.rs b/src/utils.rs index ec66f0e2..1fdefe81 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -21,13 +21,7 @@ use std::convert::TryInto; pub fn get_current_depth(target: &str) -> usize { log::trace!("enter: get_current_depth({})", target); - let target = if !target.ends_with('/') { - // target url doesn't end with a /, for the purposes of determining depth, we'll normalize - // all urls to end in a / and then calculate accordingly - format!("{}/", target) - } else { - String::from(target) - }; + let target = normalize_url(target); match Url::parse(&target) { Ok(url) => { From 2b7392735aed8512c685c76bf012adc11815c987 Mon Sep 17 00:00:00 2001 From: epi Date: Fri, 13 Nov 2020 17:17:36 -0600 Subject: [PATCH 05/18] added pretty print of current scans --- src/scan_manager.rs | 49 +++++++++++++++++++++++++++++++++------------ 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/src/scan_manager.rs b/src/scan_manager.rs index 2c79a7c8..dba6d8e3 100644 --- a/src/scan_manager.rs +++ b/src/scan_manager.rs @@ -1,6 +1,8 @@ use crate::{config::PROGRESS_PRINTER, progress, SLEEP_DURATION, scanner::NUMBER_OF_REQUESTS}; +use console::style; use indicatif::ProgressBar; use std::{ + fmt, cmp::PartialEq, sync::{Arc, Mutex}, }; @@ -86,8 +88,19 @@ impl FeroxScan { } } -// /// Eq implementation -// impl Eq for FeroxScan {} +/// Display implementation +impl fmt::Display for FeroxScan { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let complete = if self.complete { + style("complete").green() + } else { + style("incomplete").red() + }; + + write!(f, "{:10} {}", complete, self.url) + } +} + /// PartialEq implementation; uses FeroxScan.id for comparison impl PartialEq for FeroxScan { @@ -162,23 +175,32 @@ impl FeroxScans { /// Find and return a `FeroxScan` based on the given URL pub fn get_scan_by_url(&self, url: &str) -> Option>> { - match self.scans.lock() { - Ok(scans) => { - for scan in scans.iter() { - if let Ok(locked_scan) = scan.lock() { - if locked_scan.url == url { - return Some(scan.clone()); - } + if let Ok(scans) = self.scans.lock() { + for scan in scans.iter() { + if let Ok(locked_scan) = scan.lock() { + if locked_scan.url == url { + return Some(scan.clone()); } } } - Err(e) => { - log::error!("FeroxScans' container's mutex is poisoned: {}", e); - } } None } + /// todo doc + pub fn display_scans(&self) { + if let Ok(scans) = self.scans.lock() { + for (i, scan) in scans.iter().enumerate() { + let msg = format!( + "{:3}: {}", + i, + scan.lock().unwrap() + ); + PROGRESS_PRINTER.println(format!("{}", msg)); + } + } + } + /// Forced the calling thread into a busy loop /// /// Every `SLEEP_DURATION` milliseconds, the function examines the result stored in `PAUSE_SCAN` @@ -199,7 +221,8 @@ impl FeroxScans { if INTERACTIVE_BARRIER.load(Ordering::Relaxed) == 0 { INTERACTIVE_BARRIER.fetch_add(1, Ordering::Relaxed); - PROGRESS_PRINTER.println(format!("Here's your shit: {:?}", self.scans)); + self.display_scans(); + let mut s = String::new(); std::io::stdin().read_line(&mut s).unwrap(); PROGRESS_PRINTER.println(format!("Here's your shit: {}", s)); From 880e884dea214cd5fde5540ef69391b0ac29390e Mon Sep 17 00:00:00 2001 From: epi Date: Tue, 17 Nov 2020 20:17:24 -0600 Subject: [PATCH 06/18] clippy and fmt --- src/main.rs | 2 +- src/scan_manager.rs | 33 ++++++++++++++------------------- src/scanner.rs | 1 - 3 files changed, 15 insertions(+), 21 deletions(-) diff --git a/src/main.rs b/src/main.rs index 6942f874..c1948ffb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,8 +3,8 @@ use feroxbuster::{ banner, config::{CONFIGURATION, PROGRESS_BAR, PROGRESS_PRINTER}, heuristics, logger, reporter, - scanner::{self, scan_url}, scan_manager::PAUSE_SCAN, + scanner::{self, scan_url}, utils::{ferox_print, get_current_depth, module_colorizer, status_colorizer}, FeroxError, FeroxResponse, FeroxResult, SLEEP_DURATION, VERSION, }; diff --git a/src/scan_manager.rs b/src/scan_manager.rs index dba6d8e3..851055b1 100644 --- a/src/scan_manager.rs +++ b/src/scan_manager.rs @@ -1,9 +1,9 @@ -use crate::{config::PROGRESS_PRINTER, progress, SLEEP_DURATION, scanner::NUMBER_OF_REQUESTS}; +use crate::{config::PROGRESS_PRINTER, progress, scanner::NUMBER_OF_REQUESTS, SLEEP_DURATION}; use console::style; use indicatif::ProgressBar; use std::{ - fmt, cmp::PartialEq, + fmt, sync::{Arc, Mutex}, }; use std::{ @@ -101,7 +101,6 @@ impl fmt::Display for FeroxScan { } } - /// PartialEq implementation; uses FeroxScan.id for comparison impl PartialEq for FeroxScan { fn eq(&self, other: &Self) -> bool { @@ -191,12 +190,8 @@ impl FeroxScans { pub fn display_scans(&self) { if let Ok(scans) = self.scans.lock() { for (i, scan) in scans.iter().enumerate() { - let msg = format!( - "{:3}: {}", - i, - scan.lock().unwrap() - ); - PROGRESS_PRINTER.println(format!("{}", msg)); + let msg = format!("{:3}: {}", i, scan.lock().unwrap()); + PROGRESS_PRINTER.println(msg); } } } @@ -225,7 +220,8 @@ impl FeroxScans { let mut s = String::new(); std::io::stdin().read_line(&mut s).unwrap(); - PROGRESS_PRINTER.println(format!("Here's your shit: {}", s)); + // todo actual logic for the scanning + PROGRESS_PRINTER.println(format!("Got {} from stdin", s.strip_suffix('\n').unwrap())); } loop { @@ -277,9 +273,9 @@ mod tests { /// a new one will be created, taking the if branch within the function async fn scanner_pause_scan_with_finished_spinner() { let now = time::Instant::now(); + let urls = FeroxScans::default(); PAUSE_SCAN.store(true, Ordering::Relaxed); - // BARRIER.write().unwrap().finish_and_clear(); let expected = time::Duration::from_secs(2); @@ -288,7 +284,7 @@ mod tests { PAUSE_SCAN.store(false, Ordering::Relaxed); }); - pause_scan().await; + urls.pause().await; assert!(now.elapsed() > expected); } @@ -298,7 +294,7 @@ mod tests { fn add_url_to_list_of_scanned_urls_with_unknown_url() { let urls = FeroxScans::default(); let url = "http://unknown_url"; - let (result, _scan) = add_url_to_list_of_scanned_urls(url, &urls); + let (result, _scan) = urls.add_scan(url); assert_eq!(result, true); } @@ -308,11 +304,11 @@ mod tests { let urls = FeroxScans::default(); let pb = ProgressBar::new(1); let url = "http://unknown_url/"; - let mut scan = FeroxScan::new(url, pb); + let scan = FeroxScan::new(url, pb); assert_eq!(urls.insert(scan), true); - let (result, _scan) = add_url_to_list_of_scanned_urls(url, &urls); + let (result, _scan) = urls.add_scan(url); assert_eq!(result, false); } @@ -323,13 +319,12 @@ mod tests { let urls = FeroxScans::default(); let pb = ProgressBar::new(1); let url = "http://unknown_url"; - let mut scan = FeroxScan::new(url, pb); + let scan = FeroxScan::new(url, pb); assert_eq!(urls.insert(scan), true); - let (result, _scan) = add_url_to_list_of_scanned_urls(url, &urls); + let (result, _scan) = urls.add_scan(url); assert_eq!(result, false); } - -} \ No newline at end of file +} diff --git a/src/scanner.rs b/src/scanner.rs index f686eb7c..c372b08d 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -666,7 +666,6 @@ pub fn initialize( #[cfg(test)] mod tests { use super::*; - use indicatif::ProgressBar; #[test] /// sending url + word without any extensions should get back one url with the joined word From 805f02ad2df86f2ef4ff53c36ef66c1b097cfae5 Mon Sep 17 00:00:00 2001 From: epi Date: Thu, 19 Nov 2020 06:45:08 -0600 Subject: [PATCH 07/18] incremental save; a transmitter isnt being dropped --- src/main.rs | 2 +- src/scan_manager.rs | 3 ++- src/scanner.rs | 4 ++-- tests/test_extractor.rs | 4 ++++ 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/main.rs b/src/main.rs index c1948ffb..a9840585 100644 --- a/src/main.rs +++ b/src/main.rs @@ -251,7 +251,7 @@ async fn clean_up( save_output: bool, ) { log::trace!( - "enter: clean_up({:?}, {:?}, {:?}, {:?}, {}", + "enter: clean_up({:?}, {:?}, {:?}, {:?}, {})", tx_term, term_handle, tx_file, diff --git a/src/scan_manager.rs b/src/scan_manager.rs index 851055b1..168ee5a9 100644 --- a/src/scan_manager.rs +++ b/src/scan_manager.rs @@ -83,6 +83,7 @@ impl FeroxScan { /// Mark the scan as complete and stop the scan's progress bar pub fn finish(&mut self) { + PROGRESS_PRINTER.println(format!("{:?} complete? {}", self, self.complete)); self.complete = true; self.stop_progress_bar(); } @@ -265,7 +266,7 @@ impl FeroxScans { mod tests { use super::*; - // todo add_url_* and pause_scan tests need to be redone + // todo scanner_pause_scan_with_finished_spinner test need to be redone #[tokio::test(core_threads = 1)] /// tests that pause_scan pauses execution and releases execution when PAUSE_SCAN is toggled diff --git a/src/scanner.rs b/src/scanner.rs index c372b08d..31d166af 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -396,7 +396,7 @@ async fn make_requests( let new_links = get_links(&ferox_response).await; for new_link in new_links { - let (unknown, _) = SCANNED_URLS.add_scan(&new_link); + let (unknown, new_scan) = SCANNED_URLS.add_scan(&new_link); if !unknown { // not unknown, i.e. we've seen the url before and don't need to scan again @@ -615,7 +615,7 @@ pub async fn scan_url( futures::future::join_all(recurser.await.unwrap()).await; log::trace!("done awaiting recursive scan receiver/scans"); - log::trace!("exit: scan_url"); + log::error!("exit: scan_url"); } /// Perform steps necessary to run scans that only need to be performed once (warming up the diff --git a/tests/test_extractor.rs b/tests/test_extractor.rs index fb5f7a36..d0f158b0 100644 --- a/tests/test_extractor.rs +++ b/tests/test_extractor.rs @@ -156,6 +156,10 @@ fn extractor_finds_same_relative_url_twice() -> Result<(), Box Date: Thu, 19 Nov 2020 08:57:33 -0600 Subject: [PATCH 08/18] removed prints from tests --- tests/test_extractor.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/test_extractor.rs b/tests/test_extractor.rs index d0f158b0..fb5f7a36 100644 --- a/tests/test_extractor.rs +++ b/tests/test_extractor.rs @@ -156,10 +156,6 @@ fn extractor_finds_same_relative_url_twice() -> Result<(), Box Date: Fri, 20 Nov 2020 14:03:23 -0600 Subject: [PATCH 09/18] fixed the hanging issue; cleaned up --- src/scan_manager.rs | 128 ++++++++++++++++++++++++++++++-------------- src/scanner.rs | 53 +++++++++--------- 2 files changed, 113 insertions(+), 68 deletions(-) diff --git a/src/scan_manager.rs b/src/scan_manager.rs index 168ee5a9..f16bfedd 100644 --- a/src/scan_manager.rs +++ b/src/scan_manager.rs @@ -20,6 +20,13 @@ static INTERACTIVE_BARRIER: AtomicUsize = AtomicUsize::new(0); /// Atomic boolean flag, used to determine whether or not a scan should pause or resume pub static PAUSE_SCAN: AtomicBool = AtomicBool::new(false); +/// Simple enum used to flag a `FeroxScan` as likely a directory or file +#[derive(Debug)] +pub enum ScanType { + File, + Directory, +} + /// Struct to hold scan-related state /// /// The purpose of this container is to open up the pathway to aborting currently running tasks and @@ -32,6 +39,9 @@ pub struct FeroxScan { /// The URL that to be scanned pub url: String, + /// The type of scan + pub scan_type: ScanType, + /// Whether or not this scan has completed pub complete: bool, @@ -58,10 +68,11 @@ impl FeroxScan { FeroxScan { id: new_id, + task: None, complete: false, url: String::new(), - task: None, progress_bar: None, + scan_type: ScanType::File, } } @@ -73,17 +84,18 @@ impl FeroxScan { } /// Given a URL and ProgressBar, create a new FeroxScan, wrap it in an Arc and return it - pub fn new(url: &str, pb: ProgressBar) -> Arc> { + pub fn new(url: &str, scan_type: ScanType, pb: Option) -> Arc> { let mut me = Self::default(); me.url = url.to_string(); - me.progress_bar = Some(pb); + me.scan_type = scan_type; + me.progress_bar = pb; + Arc::new(Mutex::new(me)) } /// Mark the scan as complete and stop the scan's progress bar pub fn finish(&mut self) { - PROGRESS_PRINTER.println(format!("{:?} complete? {}", self, self.complete)); self.complete = true; self.stop_progress_bar(); } @@ -187,12 +199,26 @@ impl FeroxScans { None } - /// todo doc + /// Print all FeroxScans of type Directory + /// + /// Example: + /// 0: complete https://10.129.45.20 + /// 9: complete https://10.129.45.20/images + /// 10: complete https://10.129.45.20/assets pub fn display_scans(&self) { if let Ok(scans) = self.scans.lock() { for (i, scan) in scans.iter().enumerate() { - let msg = format!("{:3}: {}", i, scan.lock().unwrap()); - PROGRESS_PRINTER.println(msg); + if let Ok(unlocked_scan) = scan.lock() { + match unlocked_scan.scan_type { + ScanType::Directory => { + PROGRESS_PRINTER.println(format!("{:3}: {}", i, unlocked_scan)); + } + ScanType::File => { + // we're only interested in displaying directory scans, as those are + // the only ones that make sense to be stopped + } + } + } } } } @@ -246,13 +272,20 @@ impl FeroxScans { /// If `FeroxScans` did not already contain the scan, return true; otherwise return false /// /// Also return a reference to the new `FeroxScan` - pub fn add_scan(&self, url: &str) -> (bool, Arc>) { - let progress_bar = - progress::add_bar(&url, NUMBER_OF_REQUESTS.load(Ordering::Relaxed), false); + fn add_scan(&self, url: &str, scan_type: ScanType) -> (bool, Arc>) { + let bar = match scan_type { + ScanType::Directory => { + let progress_bar = + progress::add_bar(&url, NUMBER_OF_REQUESTS.load(Ordering::Relaxed), false); - progress_bar.reset_elapsed(); + progress_bar.reset_elapsed(); - let ferox_scan = FeroxScan::new(&url, progress_bar); + Some(progress_bar) + } + ScanType::File => None, + }; + + let ferox_scan = FeroxScan::new(&url, scan_type, bar); // If the set did not contain the scan, true is returned. // If the set did contain the scan, false is returned. @@ -260,6 +293,24 @@ impl FeroxScans { (response, ferox_scan) } + + /// Given a url, create a new `FeroxScan` and add it to `FeroxScans` as a Directory Scan + /// + /// If `FeroxScans` did not already contain the scan, return true; otherwise return false + /// + /// Also return a reference to the new `FeroxScan` + pub fn add_directory_scan(&self, url: &str) -> (bool, Arc>) { + self.add_scan(&url, ScanType::Directory) + } + + /// Given a url, create a new `FeroxScan` and add it to `FeroxScans` as a File Scan + /// + /// If `FeroxScans` did not already contain the scan, return true; otherwise return false + /// + /// Also return a reference to the new `FeroxScan` + pub fn add_file_scan(&self, url: &str) -> (bool, Arc>) { + self.add_scan(&url, ScanType::File) + } } #[cfg(test)] @@ -268,34 +319,34 @@ mod tests { // todo scanner_pause_scan_with_finished_spinner test need to be redone - #[tokio::test(core_threads = 1)] - /// tests that pause_scan pauses execution and releases execution when PAUSE_SCAN is toggled - /// the spinner used during the test has had .finish_and_clear called on it, meaning that - /// a new one will be created, taking the if branch within the function - async fn scanner_pause_scan_with_finished_spinner() { - let now = time::Instant::now(); - let urls = FeroxScans::default(); - - PAUSE_SCAN.store(true, Ordering::Relaxed); - - let expected = time::Duration::from_secs(2); - - tokio::spawn(async move { - time::delay_for(expected).await; - PAUSE_SCAN.store(false, Ordering::Relaxed); - }); - - urls.pause().await; - - assert!(now.elapsed() > expected); - } + // #[tokio::test(core_threads = 1)] + // /// tests that pause_scan pauses execution and releases execution when PAUSE_SCAN is toggled + // /// the spinner used during the test has had .finish_and_clear called on it, meaning that + // /// a new one will be created, taking the if branch within the function + // async fn scanner_pause_scan_with_finished_spinner() { + // let now = time::Instant::now(); + // let urls = FeroxScans::default(); + // + // PAUSE_SCAN.store(true, Ordering::Relaxed); + // + // let expected = time::Duration::from_secs(2); + // + // tokio::spawn(async move { + // time::delay_for(expected).await; + // PAUSE_SCAN.store(false, Ordering::Relaxed); + // }); + // + // urls.pause().await; + // + // assert!(now.elapsed() > expected); + // } #[test] /// add an unknown url to the hashset, expect true fn add_url_to_list_of_scanned_urls_with_unknown_url() { let urls = FeroxScans::default(); let url = "http://unknown_url"; - let (result, _scan) = urls.add_scan(url); + let (result, _scan) = urls.add_scan(url, ScanType::Directory); assert_eq!(result, true); } @@ -305,11 +356,11 @@ mod tests { let urls = FeroxScans::default(); let pb = ProgressBar::new(1); let url = "http://unknown_url/"; - let scan = FeroxScan::new(url, pb); + let scan = FeroxScan::new(url, ScanType::Directory, Some(pb)); assert_eq!(urls.insert(scan), true); - let (result, _scan) = urls.add_scan(url); + let (result, _scan) = urls.add_scan(url, ScanType::Directory); assert_eq!(result, false); } @@ -318,13 +369,12 @@ mod tests { /// add a known url to the hashset, without a trailing slash, expect false fn add_url_to_list_of_scanned_urls_with_known_url_without_slash() { let urls = FeroxScans::default(); - let pb = ProgressBar::new(1); let url = "http://unknown_url"; - let scan = FeroxScan::new(url, pb); + let scan = FeroxScan::new(url, ScanType::File, None); assert_eq!(urls.insert(scan), true); - let (result, _scan) = urls.add_scan(url); + let (result, _scan) = urls.add_scan(url, ScanType::File); assert_eq!(result, false); } diff --git a/src/scanner.rs b/src/scanner.rs index 2c713dd7..f3afe851 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -38,11 +38,7 @@ pub static NUMBER_OF_REQUESTS: AtomicU64 = AtomicU64::new(0); lazy_static! { /// Set of urls that have been sent to [scan_url](fn.scan_url.html), used for deduplication - static ref SCANNED_URLS: FeroxScans = FeroxScans::default(); - - // todo remove if not needed - // /// A clock spinner protected with a RwLock to allow for a single thread to use at a time - // static ref BARRIER: Arc> = Arc::new(RwLock::new(true)); + pub static ref SCANNED_URLS: FeroxScans = FeroxScans::default(); /// Vector of implementors of the FeroxFilter trait static ref FILTERS: Arc>>> = Arc::new(RwLock::new(Vec::>::new())); @@ -110,7 +106,7 @@ fn spawn_recursion_handler( let mut scans = vec![]; while let Some(resp) = recursion_channel.recv().await { - let (unknown, _ferox_scan) = SCANNED_URLS.add_scan(&resp); + let (unknown, _) = SCANNED_URLS.add_directory_scan(&resp); if !unknown { // not unknown, i.e. we've seen the url before and don't need to scan again @@ -382,13 +378,6 @@ async fn make_requests( let new_links = get_links(&ferox_response).await; for new_link in new_links { - let (unknown, _) = SCANNED_URLS.add_scan(&new_link); - - if !unknown { - // not unknown, i.e. we've seen the url before and don't need to scan again - continue; - } - // create a url based on the given command line options, continue on error let new_url = match format_url( &new_link, @@ -401,6 +390,11 @@ async fn make_requests( Err(_) => continue, }; + if SCANNED_URLS.get_scan_by_url(&new_url.to_string()).is_some() { + //we've seen the url before and don't need to scan again + continue; + } + // make the request and store the response let new_response = match make_request(&CONFIGURATION.client, &new_url).await { Ok(resp) => resp, @@ -418,6 +412,8 @@ async fn make_requests( // very likely a file, simply request and report log::debug!("Singular extraction: {}", new_ferox_response); + SCANNED_URLS.add_file_scan(&new_url.to_string()); + send_report(report_chan.clone(), new_ferox_response); continue; @@ -490,23 +486,21 @@ pub async fn scan_url( // this protection allows us to add the first scanned url to SCANNED_URLS // from within the scan_url function instead of the recursion handler - SCANNED_URLS.add_scan(&target_url); - } - - let ferox_scan = SCANNED_URLS.get_scan_by_url(&target_url); - - if ferox_scan.is_none() { - // todo probably remove this, fine for testing for now - log::error!( - "Could not find FeroxScan associated with {}; exiting scan", - target_url - ); - return; + SCANNED_URLS.add_directory_scan(&target_url); } - let ferox_scan = ferox_scan.unwrap(); + let ferox_scan = match SCANNED_URLS.get_scan_by_url(&target_url) { + Some(scan) => scan, + None => { + log::error!( + "Could not find FeroxScan associated with {}; this shouldn't happen... exiting", + target_url + ); + return; + } + }; - // todo unwrap + // todo unwrap, maybe move into the scan impl itself and just manipulate progress bars that way let progress_bar = ferox_scan .lock() .unwrap() @@ -589,8 +583,9 @@ pub async fn scan_url( // drop the current permit so the semaphore will allow another scan to proceed drop(permit); - // todo unwrap - ferox_scan.lock().unwrap().finish(); + if let Ok(mut scan) = ferox_scan.lock() { + scan.finish(); + } // manually drop tx in order for the rx task's while loops to eval to false log::trace!("dropped recursion handler's transmitter"); From 1b1190582aede2aef7f30af11415996317718b29 Mon Sep 17 00:00:00 2001 From: epi Date: Fri, 20 Nov 2020 15:38:47 -0600 Subject: [PATCH 10/18] added a test for display scans --- src/scan_manager.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/scan_manager.rs b/src/scan_manager.rs index f16bfedd..4d9665f7 100644 --- a/src/scan_manager.rs +++ b/src/scan_manager.rs @@ -378,4 +378,17 @@ mod tests { assert_eq!(result, false); } + + #[test] + /// just increasing coverage, no real expectations + fn call_display_scans() { + let urls = FeroxScans::default(); + let pb = ProgressBar::new(1); + let url = "http://unknown_url/"; + let scan = FeroxScan::new(url, ScanType::Directory, Some(pb)); + assert_eq!(urls.insert(scan), true); + + urls.display_scans(); + } + } From 46a471c8a75020d007f987d9546c6d35394ae1c8 Mon Sep 17 00:00:00 2001 From: epi Date: Fri, 20 Nov 2020 16:09:30 -0600 Subject: [PATCH 11/18] added param to pause function for testability --- src/scan_manager.rs | 59 ++++++++++++++++++++++----------------------- src/scanner.rs | 2 +- 2 files changed, 30 insertions(+), 31 deletions(-) diff --git a/src/scan_manager.rs b/src/scan_manager.rs index 4d9665f7..7151e859 100644 --- a/src/scan_manager.rs +++ b/src/scan_manager.rs @@ -229,8 +229,7 @@ impl FeroxScans { /// /// When the value stored in `PAUSE_SCAN` becomes `false`, the function returns, exiting the busy /// loop - pub async fn pause(&self) { - log::trace!("enter: pause_scan"); + pub async fn pause(&self, get_user_input: bool) { // function uses tokio::time, not std // local testing showed a pretty slow increase (less than linear) in CPU usage as # of @@ -243,12 +242,14 @@ impl FeroxScans { if INTERACTIVE_BARRIER.load(Ordering::Relaxed) == 0 { INTERACTIVE_BARRIER.fetch_add(1, Ordering::Relaxed); - self.display_scans(); + if get_user_input { + self.display_scans(); - let mut s = String::new(); - std::io::stdin().read_line(&mut s).unwrap(); - // todo actual logic for the scanning - PROGRESS_PRINTER.println(format!("Got {} from stdin", s.strip_suffix('\n').unwrap())); + let mut s = String::new(); + std::io::stdin().read_line(&mut s).unwrap(); + // todo actual logic for the scanning + PROGRESS_PRINTER.println(format!("Got {} from stdin", s.strip_suffix('\n').unwrap())); + } } loop { @@ -317,29 +318,27 @@ impl FeroxScans { mod tests { use super::*; - // todo scanner_pause_scan_with_finished_spinner test need to be redone - - // #[tokio::test(core_threads = 1)] - // /// tests that pause_scan pauses execution and releases execution when PAUSE_SCAN is toggled - // /// the spinner used during the test has had .finish_and_clear called on it, meaning that - // /// a new one will be created, taking the if branch within the function - // async fn scanner_pause_scan_with_finished_spinner() { - // let now = time::Instant::now(); - // let urls = FeroxScans::default(); - // - // PAUSE_SCAN.store(true, Ordering::Relaxed); - // - // let expected = time::Duration::from_secs(2); - // - // tokio::spawn(async move { - // time::delay_for(expected).await; - // PAUSE_SCAN.store(false, Ordering::Relaxed); - // }); - // - // urls.pause().await; - // - // assert!(now.elapsed() > expected); - // } + #[tokio::test(core_threads = 1)] + /// tests that pause_scan pauses execution and releases execution when PAUSE_SCAN is toggled + /// the spinner used during the test has had .finish_and_clear called on it, meaning that + /// a new one will be created, taking the if branch within the function + async fn scanner_pause_scan_with_finished_spinner() { + let now = time::Instant::now(); + let urls = FeroxScans::default(); + + PAUSE_SCAN.store(true, Ordering::Relaxed); + + let expected = time::Duration::from_secs(2); + + tokio::spawn(async move { + time::delay_for(expected).await; + PAUSE_SCAN.store(false, Ordering::Relaxed); + }); + + urls.pause(false).await; + + assert!(now.elapsed() > expected); + } #[test] /// add an unknown url to the hashset, expect true diff --git a/src/scanner.rs b/src/scanner.rs index f3afe851..6a110842 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -557,7 +557,7 @@ pub async fn scan_url( // for every word in the wordlist, check to see if PAUSE_SCAN is set to true // when true; enter a busy loop that only exits by setting PAUSE_SCAN back // to false - SCANNED_URLS.pause().await; + SCANNED_URLS.pause(true).await; } make_requests(&tgt, &word, base_depth, txd, txr).await }), From f8b18576aa9918fbebcfc6e358ca3ceb2eb8c369 Mon Sep 17 00:00:00 2001 From: epi Date: Fri, 20 Nov 2020 16:09:40 -0600 Subject: [PATCH 12/18] added param to pause function for testability --- src/scan_manager.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/scan_manager.rs b/src/scan_manager.rs index 7151e859..2310e71d 100644 --- a/src/scan_manager.rs +++ b/src/scan_manager.rs @@ -248,7 +248,8 @@ impl FeroxScans { let mut s = String::new(); std::io::stdin().read_line(&mut s).unwrap(); // todo actual logic for the scanning - PROGRESS_PRINTER.println(format!("Got {} from stdin", s.strip_suffix('\n').unwrap())); + PROGRESS_PRINTER + .println(format!("Got {} from stdin", s.strip_suffix('\n').unwrap())); } } @@ -389,5 +390,4 @@ mod tests { urls.display_scans(); } - } From c08180872e5fd868124c3fd059c80aafed6361ac Mon Sep 17 00:00:00 2001 From: epi Date: Fri, 20 Nov 2020 19:34:23 -0600 Subject: [PATCH 13/18] added more tests for scan_manager --- src/scan_manager.rs | 53 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/src/scan_manager.rs b/src/scan_manager.rs index 2310e71d..d88ea6f9 100644 --- a/src/scan_manager.rs +++ b/src/scan_manager.rs @@ -56,10 +56,11 @@ pub struct FeroxScan { impl FeroxScan { /// Stop a currently running scan pub fn abort(&self) { + self.stop_progress_bar(); + if let Some(_task) = &self.task { // task.abort(); todo uncomment once upgraded to tokio 0.3 } - self.stop_progress_bar(); } /// Create a default FeroxScan, populates ID with a new UUID @@ -365,6 +366,36 @@ mod tests { assert_eq!(result, false); } + #[test] + /// abort should call stop_progress_bar, marking it as finished + fn abort_stops_progress_bar() { + let pb = ProgressBar::new(1); + let url = "http://unknown_url/"; + let scan = FeroxScan::new(url, ScanType::Directory, Some(pb)); + + assert_eq!( + scan.lock() + .unwrap() + .progress_bar + .as_ref() + .unwrap() + .is_finished(), + false + ); + + scan.lock().unwrap().finish(); + + assert_eq!( + scan.lock() + .unwrap() + .progress_bar + .as_ref() + .unwrap() + .is_finished(), + true + ); + } + #[test] /// add a known url to the hashset, without a trailing slash, expect false fn add_url_to_list_of_scanned_urls_with_known_url_without_slash() { @@ -384,10 +415,30 @@ mod tests { fn call_display_scans() { let urls = FeroxScans::default(); let pb = ProgressBar::new(1); + let pb_two = ProgressBar::new(2); let url = "http://unknown_url/"; + let url_two = "http://unknown_url/fa"; let scan = FeroxScan::new(url, ScanType::Directory, Some(pb)); + let scan_two = FeroxScan::new(url_two, ScanType::Directory, Some(pb_two)); + + scan_two.lock().unwrap().finish(); // one complete, one incomplete + assert_eq!(urls.insert(scan), true); urls.display_scans(); } + + #[test] + /// ensure that PartialEq compares FeroxScan.id fields + fn partial_eq_compares_the_id_field() { + let url = "http://unknown_url/"; + let scan = FeroxScan::new(url, ScanType::Directory, None); + let scan_two = FeroxScan::new(url, ScanType::Directory, None); + + assert!(!scan.lock().unwrap().eq(&scan_two.lock().unwrap())); + + scan_two.lock().unwrap().id = scan.lock().unwrap().id.clone(); + + assert!(scan.lock().unwrap().eq(&scan_two.lock().unwrap())); + } } From 8eec5ce1d9853d0a367a65144d345329e042a8c4 Mon Sep 17 00:00:00 2001 From: epi Date: Fri, 20 Nov 2020 19:53:45 -0600 Subject: [PATCH 14/18] even more tests! --- src/scan_manager.rs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/scan_manager.rs b/src/scan_manager.rs index d88ea6f9..340a4570 100644 --- a/src/scan_manager.rs +++ b/src/scan_manager.rs @@ -383,7 +383,7 @@ mod tests { false ); - scan.lock().unwrap().finish(); + scan.lock().unwrap().abort(); assert_eq!( scan.lock() @@ -441,4 +441,21 @@ mod tests { assert!(scan.lock().unwrap().eq(&scan_two.lock().unwrap())); } + + #[test] + /// test struct defaults + fn ferox_scan_defaults_are_correct() { + let scan = FeroxScan::default(); + assert!(scan.progress_bar.is_none()); + assert!(Uuid::parse_str(&scan.id).is_ok()); + assert_eq!(scan.url, String::new()); + match scan.scan_type { + ScanType::File => { + // do nothing, i.e. this is what we expect to see + } + ScanType::Directory => panic!(), + } + assert!(scan.task.is_none()); + assert_eq!(scan.complete, false); + } } From 697a1cf7151f573773b318b99a90ed9234940bed Mon Sep 17 00:00:00 2001 From: epi Date: Fri, 20 Nov 2020 20:39:18 -0600 Subject: [PATCH 15/18] added spinner back in; updated comments with what to change for 107 finalization --- Cargo.toml | 2 +- src/scan_manager.rs | 74 ++++++++++++++++++++++++++++++++++++++++----- src/scanner.rs | 4 ++- 3 files changed, 70 insertions(+), 10 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 14097fbd..7882862f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "feroxbuster" -version = "1.6.3" +version = "1.6.4" authors = ["Ben 'epi' Risher "] license = "MIT" edition = "2018" diff --git a/src/scan_manager.rs b/src/scan_manager.rs index 340a4570..b2f8b100 100644 --- a/src/scan_manager.rs +++ b/src/scan_manager.rs @@ -1,10 +1,11 @@ use crate::{config::PROGRESS_PRINTER, progress, scanner::NUMBER_OF_REQUESTS, SLEEP_DURATION}; use console::style; -use indicatif::ProgressBar; +use indicatif::{ProgressBar, ProgressStyle}; +use lazy_static::lazy_static; use std::{ cmp::PartialEq, fmt, - sync::{Arc, Mutex}, + sync::{Arc, Mutex, RwLock}, }; use std::{ io::{stderr, Write}, @@ -13,6 +14,12 @@ use std::{ use tokio::{task::JoinHandle, time}; use uuid::Uuid; +lazy_static! { + /// A clock spinner protected with a RwLock to allow for a single thread to use at a time + // todo remove this when issue #107 is resolved + static ref SINGLE_SPINNER: RwLock = RwLock::new(get_single_spinner()); +} + /// Single atomic number that gets incremented once, used to track first thread to interact with /// when pausing a scan static INTERACTIVE_BARRIER: AtomicUsize = AtomicUsize::new(0); @@ -246,24 +253,47 @@ impl FeroxScans { if get_user_input { self.display_scans(); - let mut s = String::new(); - std::io::stdin().read_line(&mut s).unwrap(); - // todo actual logic for the scanning - PROGRESS_PRINTER - .println(format!("Got {} from stdin", s.strip_suffix('\n').unwrap())); + let mut user_input = String::new(); + std::io::stdin().read_line(&mut user_input).unwrap(); + // todo actual logic for parsing user input in a way that allows for + // calling .abort on the scan retrieved based on the input (issue #107) + } + } + + if SINGLE_SPINNER.read().unwrap().is_finished() { + // todo remove this when issue #107 is resolved + + // in order to not leave draw artifacts laying around in the terminal, we call + // finish_and_clear on the progress bar when resuming scans. For this reason, we need to + // check if the spinner is finished, and repopulate the RwLock with a new spinner if + // necessary + if let Ok(mut guard) = SINGLE_SPINNER.write() { + *guard = get_single_spinner(); } } + if let Ok(spinner) = SINGLE_SPINNER.write() { + spinner.enable_steady_tick(120); + } + loop { // first tick happens immediately, all others wait the specified duration interval.tick().await; if !PAUSE_SCAN.load(Ordering::Acquire) { // PAUSE_SCAN is false, so we can exit the busy loop - let _ = stderr().flush(); + if INTERACTIVE_BARRIER.load(Ordering::Relaxed) == 1 { INTERACTIVE_BARRIER.fetch_sub(1, Ordering::Relaxed); } + + if let Ok(spinner) = SINGLE_SPINNER.write() { + // todo remove this when issue #107 is resolved + spinner.finish_and_clear(); + } + + let _ = stderr().flush(); + log::trace!("exit: pause_scan"); return; } @@ -316,10 +346,38 @@ impl FeroxScans { } } +/// Return a clock spinner, used when scans are paused +// todo remove this when issue #107 is resolved +fn get_single_spinner() -> ProgressBar { + log::trace!("enter: get_single_spinner"); + + let spinner = ProgressBar::new_spinner().with_style( + ProgressStyle::default_spinner() + .tick_strings(&[ + "🕛", "🕐", "🕑", "🕒", "🕓", "🕔", "🕕", "🕖", "🕗", "🕘", "🕙", "🕚", + ]) + .template(&format!( + "\t-= All Scans {{spinner}} {} =-", + style("Paused").red() + )), + ); + + log::trace!("exit: get_single_spinner -> {:?}", spinner); + spinner +} + #[cfg(test)] mod tests { use super::*; + #[test] + /// test that get_single_spinner returns the correct spinner + // todo remove this when issue #107 is resolved + fn scanner_get_single_spinner_returns_spinner() { + let spinner = get_single_spinner(); + assert!(!spinner.is_finished()); + } + #[tokio::test(core_threads = 1)] /// tests that pause_scan pauses execution and releases execution when PAUSE_SCAN is toggled /// the spinner used during the test has had .finish_and_clear called on it, meaning that diff --git a/src/scanner.rs b/src/scanner.rs index 6a110842..3e269eee 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -557,7 +557,9 @@ pub async fn scan_url( // for every word in the wordlist, check to see if PAUSE_SCAN is set to true // when true; enter a busy loop that only exits by setting PAUSE_SCAN back // to false - SCANNED_URLS.pause(true).await; + + // todo change to true when issue #107 is resolved + SCANNED_URLS.pause(false).await; } make_requests(&tgt, &word, base_depth, txd, txr).await }), From 582ce9ed8d148fd5be60cad649b520927b2e0fa5 Mon Sep 17 00:00:00 2001 From: epi Date: Sat, 21 Nov 2020 06:40:42 -0600 Subject: [PATCH 16/18] bumped version to 1.6.3 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 7882862f..14097fbd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "feroxbuster" -version = "1.6.4" +version = "1.6.3" authors = ["Ben 'epi' Risher "] license = "MIT" edition = "2018" From df19c639011020a4834feed9705551733c7a547c Mon Sep 17 00:00:00 2001 From: epi Date: Sat, 21 Nov 2020 07:36:43 -0600 Subject: [PATCH 17/18] fixed up getting the progress bar in scanner --- src/scan_manager.rs | 29 +++++++++++++++++++++++++++++ src/scanner.rs | 16 +++++++--------- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/src/scan_manager.rs b/src/scan_manager.rs index b2f8b100..f8d14820 100644 --- a/src/scan_manager.rs +++ b/src/scan_manager.rs @@ -91,6 +91,22 @@ impl FeroxScan { } } + /// Simple helper get a progress bar + pub fn progress_bar(&mut self) -> ProgressBar { + if let Some(pb) = &self.progress_bar { + pb.clone() + } else { + let num_requests = NUMBER_OF_REQUESTS.load(Ordering::Relaxed); + let pb = progress::add_bar(&self.url, num_requests, false); + + pb.reset_elapsed(); + + self.progress_bar = Some(pb.clone()); + + pb + } + } + /// Given a URL and ProgressBar, create a new FeroxScan, wrap it in an Arc and return it pub fn new(url: &str, scan_type: ScanType, pb: Option) -> Arc> { let mut me = Self::default(); @@ -516,4 +532,17 @@ mod tests { assert!(scan.task.is_none()); assert_eq!(scan.complete, false); } + + #[test] + /// show that a new progress bar is created if one doesn't exist + fn ferox_scan_get_progress_bar_when_none_is_set() { + let mut scan = FeroxScan::default(); + + assert!(scan.progress_bar.is_none()); // no pb exists + + let pb = scan.progress_bar(); + + assert!(scan.progress_bar.is_some()); // new pb created + assert!(!pb.is_finished()) // not finished + } } diff --git a/src/scanner.rs b/src/scanner.rs index 3e269eee..271c3ab5 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -500,15 +500,13 @@ pub async fn scan_url( } }; - // todo unwrap, maybe move into the scan impl itself and just manipulate progress bars that way - let progress_bar = ferox_scan - .lock() - .unwrap() - .progress_bar - .as_ref() - .unwrap() - .clone(); - progress_bar.reset_elapsed(); + let progress_bar = match ferox_scan.lock() { + Ok(mut scan) => scan.progress_bar(), + Err(e) => { + log::error!("FeroxScan's ({:?}) mutex is poisoned: {}", ferox_scan, e); + return; + } + }; // When acquire is called and the semaphore has remaining permits, the function immediately // returns a permit. However, if no remaining permits are available, acquire (asynchronously) From 38817898791053a83510f8e0e3673c666168ab79 Mon Sep 17 00:00:00 2001 From: epi Date: Sat, 21 Nov 2020 07:55:10 -0600 Subject: [PATCH 18/18] removed unnecessary test --- src/scan_manager.rs | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/src/scan_manager.rs b/src/scan_manager.rs index f8d14820..7c3c1384 100644 --- a/src/scan_manager.rs +++ b/src/scan_manager.rs @@ -516,23 +516,6 @@ mod tests { assert!(scan.lock().unwrap().eq(&scan_two.lock().unwrap())); } - #[test] - /// test struct defaults - fn ferox_scan_defaults_are_correct() { - let scan = FeroxScan::default(); - assert!(scan.progress_bar.is_none()); - assert!(Uuid::parse_str(&scan.id).is_ok()); - assert_eq!(scan.url, String::new()); - match scan.scan_type { - ScanType::File => { - // do nothing, i.e. this is what we expect to see - } - ScanType::Directory => panic!(), - } - assert!(scan.task.is_none()); - assert_eq!(scan.complete, false); - } - #[test] /// show that a new progress bar is created if one doesn't exist fn ferox_scan_get_progress_bar_when_none_is_set() {