From 65ea28394a95eb40d98d6e29ddecbf397878605b Mon Sep 17 00:00:00 2001 From: Zach Carlson Date: Sun, 3 Dec 2023 16:00:33 -0600 Subject: [PATCH 01/26] Update deps for nested crate... --- crates/rusty-hog-scanner/Cargo.toml | 8 ++++---- crates/rusty-hog-scanner/src/lib.rs | 25 +++++++++++++------------ 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/crates/rusty-hog-scanner/Cargo.toml b/crates/rusty-hog-scanner/Cargo.toml index 0519211..964b533 100644 --- a/crates/rusty-hog-scanner/Cargo.toml +++ b/crates/rusty-hog-scanner/Cargo.toml @@ -7,11 +7,11 @@ edition = "2021" serde = "1.0" serde_json = "1.0" serde_derive = "^1" -clap = "2" -simple_logger = "1.11" -simple-error = "0.2" +clap = "4" +simple_logger = "4.3" +simple-error = "0.3" anyhow = "1.0" log = "0.4" -base64 = "0.13" +base64 = "0.21" regex = "1" hex = "0.4" diff --git a/crates/rusty-hog-scanner/src/lib.rs b/crates/rusty-hog-scanner/src/lib.rs index 3dc998a..e24f1f5 100644 --- a/crates/rusty-hog-scanner/src/lib.rs +++ b/crates/rusty-hog-scanner/src/lib.rs @@ -65,10 +65,11 @@ //! assert_eq!(secrets.pop().unwrap(), "Email address"); //! ``` -#[macro_use] +//#[macro_use] extern crate clap; use anyhow::Result; +use base64::{Engine as _, engine::general_purpose as Base64Engine}; use clap::ArgMatches; use log::{self, debug, error, info, LevelFilter}; use regex::bytes::{Match, Matches, Regex, RegexBuilder}; @@ -372,26 +373,26 @@ impl SecretScannerBuilder { /// Configure multiple values using the clap library's `ArgMatches` object. /// This function looks for a "CASE" flag and "REGEX", "ALLOWLIST", "DEFAULT_ENTROPY_THRESHOLD" values. pub fn conf_argm(mut self, arg_matches: &ArgMatches) -> Self { - self.case_insensitive = arg_matches.is_present("CASE"); - self.regex_json_path = match arg_matches.value_of("REGEX") { + self.case_insensitive = arg_matches.get_flag("CASE"); + self.regex_json_path = match arg_matches.get_one::("REGEX") { Some(s) => Some(String::from(s)), None => None, }; - self.pretty_print = arg_matches.is_present("PRETTYPRINT"); - self.output_path = match arg_matches.value_of("OUTPUT") { + self.pretty_print = arg_matches.get_flag("PRETTYPRINT"); + self.output_path = match arg_matches.get_one::("OUTPUT") { Some(s) => Some(String::from(s)), None => None, }; - self.allowlist_json_path = match arg_matches.value_of("ALLOWLIST") { + self.allowlist_json_path = match arg_matches.get_one::("ALLOWLIST") { Some(s) => Some(String::from(s)), None => None, }; self.default_entropy_threshold = - match value_t!(arg_matches.value_of("DEFAULT_ENTROPY_THRESHOLD"), f32) { - Ok(t) => t, - Err(_) => DEFAULT_ENTROPY_THRESHOLD, + match arg_matches.get_one::("DEFAULT_ENTROPY_THRESHOLD") { + Some(t) => *t, + None => DEFAULT_ENTROPY_THRESHOLD, }; - self.add_entropy_findings = arg_matches.is_present("ENTROPY"); + self.add_entropy_findings = arg_matches.get_flag("ENTROPY"); self } @@ -838,11 +839,11 @@ impl SecretScanner { let b64_words: Vec = words .iter() .filter(|word| word.len() >= 20 && Self::is_base64_string(word)) - .filter_map(|x| base64::decode(x).ok()) + .filter_map(|x| Base64Engine::STANDARD_NO_PAD.decode(x).ok()) .filter(|word| { Self::calc_normalized_entropy(word, Some(255), false) > entropy_threshold }) - .map(|word| String::from(base64::encode(&word).as_str())) + .map(|word| String::from(Base64Engine::STANDARD_NO_PAD.encode(&word).as_str())) .collect(); let hex_words: Vec = words .iter() // there must be a better way From 22272ce404697875ab684328252f8c4d61d51cfe Mon Sep 17 00:00:00 2001 From: Zach Carlson Date: Sun, 3 Dec 2023 16:48:23 -0600 Subject: [PATCH 02/26] rusty-hogs dep update: clap: duroc_hog --- Cargo.toml | 2 +- src/bin/duroc_hog.rs | 44 ++++++++++++++++++++++---------------------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0f108b0..c21bb96 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ git2 = "0.13" serde = "1.0" serde_json = "1.0" serde_derive = "^1" -clap = "2" +clap = "4" regex = "1" url = "2" tempdir = "0.3" diff --git a/src/bin/duroc_hog.rs b/src/bin/duroc_hog.rs index ec90e7d..38e0b52 100644 --- a/src/bin/duroc_hog.rs +++ b/src/bin/duroc_hog.rs @@ -34,7 +34,7 @@ extern crate chrono; extern crate encoding; -use clap::ArgMatches; +use clap::{Arg, ArgAction, ArgMatches, Command}; use log::{self, debug, error, info}; use serde::{Deserialize, Serialize}; use simple_error::SimpleError; @@ -68,22 +68,21 @@ const GZEXTENSIONS: &[&str] = &["gz", "tgz"]; /// Main entry function that uses the [clap crate](https://docs.rs/clap/2.33.0/clap/) fn main() { - let matches = clap_app!(duroc_hog => - (version: "1.0.11") - (author: "Scott Cutler ") - (about: "File system secret scanner in Rust") - (@arg REGEX: -r --regex +takes_value "Sets a custom regex JSON file") - (@arg FSPATH: +required "Sets the path of the directory or file to scan.") - (@arg NORECURSIVE: --norecursive "Disable recursive scanning of all subdirectories underneath the supplied path") - (@arg VERBOSE: -v --verbose ... "Sets the level of debugging information") - (@arg ENTROPY: --entropy ... "Enables entropy scanning") - (@arg DEFAULT_ENTROPY_THRESHOLD: --default_entropy_threshold +takes_value "Default entropy threshold (0.6 by default)") - (@arg UNZIP: -z --unzip "Recursively scans archives (ZIP and TAR) in memory (dangerous)") - (@arg CASE: --caseinsensitive "Sets the case insensitive flag for all regexes") - (@arg OUTPUT: -o --outputfile +takes_value "Sets the path to write the scanner results to (stdout by default)") - (@arg PRETTYPRINT: --prettyprint "Outputs the JSON in human readable format") - (@arg ALLOWLIST: -a --allowlist +takes_value "Sets a custom allowlist JSON file") - ) + let matches = Command::new("duroc_hog") + .version("1.0.11") + .author("Scott Cutler ") + .about("File system secret scanner in Rust") + .arg(Arg::new("REGEX").short('r').long("regex").action(ArgAction::Set).value_name("REGEX").help("Sets a custom regex JSON file")) + .arg(Arg::new("FSPATH").required(true).action(ArgAction::Set).value_name("PATH").help("Sets the path of the directory or file to scan.")) + .arg(Arg::new("NORECURSIVE").long("norecursive").action(ArgAction::SetTrue).help("Disable recursive scanning of all subdirectories underneath the supplied path")) + .arg(Arg::new("VERBOSE").short('v').long("verbose").action(ArgAction::Count).help("Sets the level of debugging information")) + .arg(Arg::new("ENTROPY").long("entropy").action(ArgAction::SetTrue).help("Enables entropy scanning")) + .arg(Arg::new("DEFAULT_ENTROPY_THRESHOLD").long("default_entropy_threshold").action(ArgAction::Set).default_value("0.6").help("Default entropy threshold (0.6 by default)")) + .arg(Arg::new("UNZIP").short('z').long("unzip").action(ArgAction::SetTrue).help("Recursively scans archives (ZIP and TAR) in memory (dangerous)")) + .arg(Arg::new("CASE").long("caseinsensitive").action(ArgAction::SetTrue).help("Sets the case insensitive flag for all regexes")) + .arg(Arg::new("OUTPUT").short('o').long("outputfile").action(ArgAction::Set).help("Sets the path to write the scanner results to (stdout by default)")) + .arg(Arg::new("PRETTYPRINT").long("prettyprint").action(ArgAction::SetTrue).help("Outputs the JSON in human readable format")) + .arg(Arg::new("ALLOWLIST").short('a').long("allowlist").action(ArgAction::Set).help("Sets a custom allowlist JSON file")) .get_matches(); match run(&matches) { Ok(()) => {} @@ -94,15 +93,16 @@ fn main() { /// Main logic contained here. Get the CLI variables, and use them to initialize a GitScanner fn run(arg_matches: &ArgMatches) -> Result<(), SimpleError> { // Set logging - SecretScanner::set_logging(arg_matches.occurrences_of("VERBOSE")); + SecretScanner::set_logging(arg_matches.get_count("VERBOSE").into()); // Initialize some more variables let secret_scanner = SecretScannerBuilder::new().conf_argm(arg_matches).build(); // let scan_entropy = arg_matches.is_present("ENTROPY"); - let recursive = !arg_matches.is_present("NORECURSIVE"); - let fspath = Path::new(arg_matches.value_of("FSPATH").unwrap()); - let output_file = Path::new(arg_matches.value_of("OUTPUT").unwrap_or("")); - let unzip: bool = arg_matches.is_present("UNZIP"); + let recursive = !arg_matches.get_flag("NORECURSIVE"); + let fspath = Path::new(arg_matches.get_one::("FSPATH").unwrap()); + let default_path = String::from(""); + let output_file = Path::new(arg_matches.get_one("OUTPUT").unwrap_or(&default_path)); + let unzip: bool = arg_matches.get_flag("UNZIP"); debug!("fspath: {:?}", fspath); From 6eba5c7f8225a2ffcebcb9f50ce7241e8131b342 Mon Sep 17 00:00:00 2001 From: Zach Carlson Date: Sun, 3 Dec 2023 17:26:52 -0600 Subject: [PATCH 03/26] rusty-hogs dep update: clap: choctaw_hog --- src/bin/choctaw_hog.rs | 71 +++++++++++++++++++++--------------------- 1 file changed, 35 insertions(+), 36 deletions(-) diff --git a/src/bin/choctaw_hog.rs b/src/bin/choctaw_hog.rs index 5ae54b6..116fccc 100644 --- a/src/bin/choctaw_hog.rs +++ b/src/bin/choctaw_hog.rs @@ -40,7 +40,7 @@ extern crate chrono; extern crate encoding; -use clap::ArgMatches; +use clap::{Arg, ArgAction, ArgMatches, Command}; use log::{self, error, info}; use simple_error::SimpleError; use std::str; @@ -51,28 +51,27 @@ use rusty_hog_scanner::{SecretScanner, SecretScannerBuilder}; /// Main entry function that uses the [clap crate](https://docs.rs/clap/2.33.0/clap/) fn main() { - let matches = clap_app!(choctaw_hog => - (version: "1.0.11") - (author: "Scott Cutler ") - (about: "Git secret scanner in Rust") - (@arg REGEX: -r --regex +takes_value "Sets a custom regex JSON file") - (@arg GITPATH: +required "Sets the path (or URL) of the Git repo to scan. SSH links must include username (git@)") - (@arg VERBOSE: -v --verbose ... "Sets the level of debugging information") - (@arg ENTROPY: --entropy ... "Enables entropy scanning") - (@arg DEFAULT_ENTROPY_THRESHOLD: --default_entropy_threshold +takes_value "Default entropy threshold (0.6 by default)") - (@arg CASE: --caseinsensitive "Sets the case insensitive flag for all regexes") - (@arg OUTPUT: -o --outputfile +takes_value "Sets the path to write the scanner results to (stdout by default)") - (@arg PRETTYPRINT: --prettyprint "Outputs the JSON in human readable format") - (@arg SINCECOMMIT: --since_commit +takes_value "Filters commits based on date committed (branch agnostic)") - (@arg UNTILCOMMIT: --until_commit +takes_value "Filters commits based on date committed (branch agnostic)") - (@arg SSHKEYPATH: --sshkeypath +takes_value "Takes a path to a private SSH key for git authentication, defaults to ssh-agent") - (@arg SSHKEYPHRASE: --sshkeyphrase +takes_value "Takes a passphrase to a private SSH key for git authentication, defaults to none") - (@arg HTTPSUSER: --httpsuser +takes_value "Takes a username for HTTPS-based authentication") - (@arg HTTPSPASS: --httpspass +takes_value "Takes a password for HTTPS-based authentication") - (@arg RECENTDAYS: --recent_days +takes_value conflicts_with[SINCECOMMIT] "Filters commits to the last number of days (branch agnostic)") - (@arg ALLOWLIST: -a --allowlist +takes_value "Sets a custom allowlist JSON file") - ) - .get_matches(); + let matches = Command::new("choctaw_hog") + .version("1.0.11") + .author("Scott Cutler ") + .about("Git secret scanner in Rust") + .arg(Arg::new("REGEX").short('r').long("regex").action(ArgAction::Set).value_name("REGEX").help("Sets a custom regex JSON file")) + .arg(Arg::new("GITPATH").required(true).action(ArgAction::Set).value_name("GIT_PATH").help("Sets the path (or URL) of the Git repo to scan. SSH links must include username (git@)")) + .arg(Arg::new("VERBOSE").short('v').long("verbose").action(ArgAction::Count).help("Sets the level of debugging information")) + .arg(Arg::new("ENTROPY").long("entropy").action(ArgAction::SetTrue).help("Enables entropy scanning")) + .arg(Arg::new("DEFAULT_ENTROPY_THRESHOLD").long("default_entropy_threshold").action(ArgAction::Set).default_value("0.6").help("Default entropy threshold (0.6 by default)")) + .arg(Arg::new("CASE").long("caseinsensitive").action(ArgAction::SetTrue).help("Sets the case insensitive flag for all regexes")) + .arg(Arg::new("OUTPUT").short('o').long("outputfile").action(ArgAction::Set).help("Sets the path to write the scanner results to (stdout by default)")) + .arg(Arg::new("PRETTYPRINT").long("prettyprint").action(ArgAction::SetTrue).help("Outputs the JSON in human readable format")) + .arg(Arg::new("SINCECOMMIT").long("since_commit").action(ArgAction::Set).help("Filters commits based on date committed (branch agnostic)")) + .arg(Arg::new("UNTILCOMMIT").long("until_commit").action(ArgAction::Set).help("Filters commits based on date committed (branch agnostic)")) + .arg(Arg::new("SSHKEYPATH").long("sshkeypath").action(ArgAction::Set).help("Takes a path to a private SSH key for git authentication, defaults to ssh-agent")) + .arg(Arg::new("SSHKEYPHRASE").long("sshkeyphrase").action(ArgAction::Set).help("Takes a passphrase to a private SSH key for git authentication, defaults to none")) + .arg(Arg::new("HTTPSUSER").long("httpsuser").action(ArgAction::Set).help("Takes a username for HTTPS-based authentication")) + .arg(Arg::new("HTTPSPASS").long("httpspass").action(ArgAction::Set).help("Takes a password for HTTPS-based authentication")) + .arg(Arg::new("RECENTDAYS").long("recent_days").action(ArgAction::Set).conflicts_with("SINCECOMMIT").help("Filters commits to the last number of days (branch agnostic)")) + .arg(Arg::new("ALLOWLIST").short('a').long("allowlist").action(ArgAction::Set).help("Sets a custom allowlist JSON file")) + .get_matches(); match run(&matches) { Ok(()) => {} Err(e) => error!("Error running command: {}", e), @@ -82,31 +81,31 @@ fn main() { /// Main logic contained here. Get the CLI variables, and use them to initialize a GitScanner fn run(arg_matches: &ArgMatches) -> Result<(), SimpleError> { // Set logging - SecretScanner::set_logging(arg_matches.occurrences_of("VERBOSE")); + SecretScanner::set_logging(arg_matches.get_count("VERBOSE").into()); // Initialize some more variables let secret_scanner = SecretScannerBuilder::new().conf_argm(arg_matches).build(); - let sshkeypath = arg_matches.value_of("SSHKEYPATH"); - let sshkeyphrase = arg_matches.value_of("SSHKEYPHRASE"); - let httpsuser = arg_matches.value_of("HTTPSUSER"); - let httpspass = arg_matches.value_of("HTTPSPASS"); - let since_commit = arg_matches.value_of("SINCECOMMIT"); - let until_commit = arg_matches.value_of("UNTILCOMMIT"); - let recent_days: Option = match value_t!(arg_matches.value_of("RECENTDAYS"), u32) { - Ok(d) => { - if d == 0 { + let sshkeypath = arg_matches.get_one::("SSHKEYPATH").map(|s| s.as_str()); + let sshkeyphrase = arg_matches.get_one::("SSHKEYPHRASE").map(|s| s.as_str()); + let httpsuser = arg_matches.get_one::("HTTPSUSER").map(|s| s.as_str()); + let httpspass = arg_matches.get_one::("HTTPSPASS").map(|s| s.as_str()); + let since_commit = arg_matches.get_one::("SINCECOMMIT").map(|s| s.as_str()); + let until_commit = arg_matches.get_one::("UNTILCOMMIT").map(|s| s.as_str()); + let recent_days: Option = match arg_matches.get_one::("RECENTDAYS") { + Some(d) => { + if *d == 0 { None } else { - Some(d) + Some(*d) } } - Err(_e) => None, + None => None, }; // Get Git objects let dest_dir = TempDir::new("rusty_hogs").unwrap(); let dest_dir_path = dest_dir.path(); - let source_path: &str = arg_matches.value_of("GITPATH").unwrap(); + let source_path: &str = arg_matches.get_one::("GITPATH").map(|s| s.as_str()).unwrap(); // Do the scan let git_scanner = GitScanner::new_from_scanner(secret_scanner).init_git_repo( From efc4f9c9fed3f2e8b71cad23f3aa1a066ba9f254 Mon Sep 17 00:00:00 2001 From: Zach Carlson Date: Sun, 3 Dec 2023 17:46:49 -0600 Subject: [PATCH 04/26] rusty-hogs dep update: clap: berkshire_hog --- src/bin/berkshire_hog.rs | 47 +++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/src/bin/berkshire_hog.rs b/src/bin/berkshire_hog.rs index e70a0f3..e0c0ad1 100644 --- a/src/bin/berkshire_hog.rs +++ b/src/bin/berkshire_hog.rs @@ -28,7 +28,7 @@ #[macro_use] extern crate clap; -use clap::ArgMatches; +use clap::{Arg, ArgAction, ArgMatches, Command}; use log::{self, debug, error, info}; use s3::bucket::Bucket; use s3::creds::Credentials; @@ -44,25 +44,22 @@ use std::collections::HashSet; /// Main entry function that uses the [clap crate](https://docs.rs/clap/2.33.0/clap/) fn main() { - let matches = clap_app!(berkshire_hog => - (version: "1.0.11") - (author: "Scott Cutler ") - (about: "S3 secret hunter in Rust. Avoid bandwidth costs, run this within a VPC!") - (@arg REGEX: --regex +takes_value "Sets a custom regex JSON file") - (@arg S3URI: +required "The location of a S3 bucket and optional prefix or filename to scan. This must be written in the form s3://mybucket[/prefix_or_file]") - (@arg S3REGION: +required "Sets the region of the S3 bucket to scan") - (@arg RECURSIVE: -r --recursive "Recursively scans files under the prefix") - (@arg VERBOSE: -v --verbose ... "Sets the level of debugging information") - (@arg ENTROPY: --entropy ... "Enables entropy scanning") - (@arg DEFAULT_ENTROPY_THRESHOLD: --default_entropy_threshold +takes_value "Default entropy threshold (0.6 by default)") - (@arg CASE: --caseinsensitive "Sets the case insensitive flag for all regexes") - (@arg OUTPUT: -o --outputfile +takes_value "Sets the path to write the scanner results to (stdout by default)") - (@arg PRETTYPRINT: --prettyprint "Outputs the JSON in human readable format") - (@arg PROFILE: --profile +takes_value "When using a configuration file, enables a non-default profile") - (@arg ALLOWLIST: -a --allowlist +takes_value "Sets a custom allowlist JSON file") -// (@arg AWS_ACCESS_KEY_ID: --awsaccesskeyid +takes_value "Forces manual AWS authentication") -// (@arg AWS_SECRET_ACCESS_KEY: --awssecretaccesskey +takes_value "Forces manual AWS authentication") - ) + let matches = Command::new("berkshire_hog") + .version("1.0.11") + .author("Scott Cutler ") + .about("S3 secret hunter in Rust. Avoid bandwidth costs, run this within a VPC!") + .arg(Arg::new("REGEX").long("regex").action(ArgAction::Set).help("Sets a custom regex JSON file")) + .arg(Arg::new("S3URI").required(true).action(ArgAction::Set).help("The location of a S3 bucket and optional prefix or filename to scan. This must be written in the form s3://mybucket[/prefix_or_file]")) + .arg(Arg::new("S3REGION").required(true).action(ArgAction::Set).help("Sets the region of the S3 bucket to scan")) + .arg(Arg::new("RECURSIVE").short('r').long("recursive").action(ArgAction::SetTrue).help("Recursively scans files under the prefix")) + .arg(Arg::new("VERBOSE").short('v').long("verbose").action(ArgAction::Count).help("Sets the level of debugging information")) + .arg(Arg::new("ENTROPY").long("entropy").action(ArgAction::SetTrue).help("Enables entropy scanning")) + .arg(Arg::new("DEFAULT_ENTROPY_THRESHOLD").long("default_entropy_threshold").action(ArgAction::Set).default_value("0.6").help("Default entropy threshold (0.6 by default)")) + .arg(Arg::new("CASE").long("caseinsensitive").action(ArgAction::SetTrue).help("Sets the case insensitive flag for all regexes")) + .arg(Arg::new("OUTPUT").short('o').long("outputfile").action(ArgAction::Set).help("Sets the path to write the scanner results to (stdout by default)")) + .arg(Arg::new("PRETTYPRINT").long("prettyprint").action(ArgAction::SetTrue).help("Outputs the JSON in human readable format")) + .arg(Arg::new("PROFILE").long("profile").action(ArgAction::Set).help("When using a configuration file, enables a non-default profile")) + .arg(Arg::new("ALLOWLIST").short('a').long("allowlist").action(ArgAction::Set).help("Sets a custom allowlist JSON file")) .get_matches(); match run(&matches) { Ok(()) => {} @@ -73,7 +70,7 @@ fn main() { /// Main logic contained here. Initialize S3Scanner, parse the URL and objects, then run the scan. fn run(arg_matches: &ArgMatches) -> Result<(), SimpleError> { // Set logging - SecretScanner::set_logging(arg_matches.occurrences_of("VERBOSE")); + SecretScanner::set_logging(arg_matches.get_count("VERBOSE").into()); // Get regex objects let ss = SecretScannerBuilder::new().conf_argm(arg_matches).build(); @@ -81,7 +78,7 @@ fn run(arg_matches: &ArgMatches) -> Result<(), SimpleError> { // Parse the S3URI let url: Url = try_with!( - Url::parse(arg_matches.value_of("S3URI").unwrap()), + Url::parse(arg_matches.get_one::("S3URI").unwrap().as_str()), "Failed to parse S3URI" ); let bucket_string = require_with!(url.host_str(), "Bucket name not detected in S3 URI"); @@ -92,13 +89,13 @@ fn run(arg_matches: &ArgMatches) -> Result<(), SimpleError> { }; // Initialize our S3 variables - let profile = arg_matches.value_of("PROFILE").map(|x| x.to_string()); + let profile = arg_matches.get_one::("PROFILE").map(|s| s.as_str()); let credentials = Credentials::new(None, None, None, None, profile.as_deref()).unwrap(); debug!( "credentials: {:?} {:?} {:?}", credentials.access_key, credentials.secret_key, credentials.security_token ); - let region_str = arg_matches.value_of("S3REGION").unwrap(); + let region_str = arg_matches.get_one::("S3REGION").unwrap(); let region: Region = match region_str.parse() { Ok(r) => r, Err(e) => return Err(SimpleError::new(e.to_string())), @@ -108,7 +105,7 @@ fn run(arg_matches: &ArgMatches) -> Result<(), SimpleError> { Err(e) => return Err(SimpleError::new(e.to_string())), }; - let delimiter = if arg_matches.is_present("RECURSIVE") { + let delimiter = if arg_matches.get_flag("RECURSIVE") { None } else { Some(String::from("/")) From aecd3fe8fd5624ee6d38a985561c52c0d032289a Mon Sep 17 00:00:00 2001 From: Zach Carlson Date: Sun, 3 Dec 2023 17:59:20 -0600 Subject: [PATCH 05/26] rusty-hogs dep update: clap: hante_hog --- src/bin/hante_hog.rs | 55 +++++++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/src/bin/hante_hog.rs b/src/bin/hante_hog.rs index 9ff0b3b..a804fe0 100644 --- a/src/bin/hante_hog.rs +++ b/src/bin/hante_hog.rs @@ -32,7 +32,7 @@ extern crate clap; extern crate hyper; extern crate hyper_rustls; -use clap::ArgMatches; +use clap::{Arg, ArgAction, ArgMatches, Command}; use encoding::all::ASCII; use encoding::types::Encoding; use encoding::DecoderTrap; @@ -65,24 +65,23 @@ pub struct SlackFinding { /// Main entry function that uses the [clap crate](https://docs.rs/clap/2.33.0/clap/) #[tokio::main] async fn main() { - let matches: ArgMatches = clap_app!(hante_hog => - (version: "1.0.11") - (author: "Joao Henrique Machado Silva ") - (about: "Slack secret scanner in Rust.") - (@arg REGEX: --regex +takes_value "Sets a custom regex JSON file") - (@arg CHANNELID: --channelid +required +takes_value "The ID (e.g. C12345) of the Slack channel you want to scan") - (@arg VERBOSE: -v --verbose ... "Sets the level of debugging information") - (@arg ENTROPY: --entropy ... "Enables entropy scanning") - (@arg DEFAULT_ENTROPY_THRESHOLD: --default_entropy_threshold +takes_value "Default entropy threshold (0.6 by default)") - (@arg CASE: --caseinsensitive "Sets the case insensitive flag for all regexes") - (@arg OUTPUT: -o --outputfile +takes_value "Sets the path to write the scanner results to (stdout by default)") - (@arg PRETTYPRINT: --prettyprint "Outputs the JSON in human readable format") - (@arg BEARERTOKEN: --authtoken +takes_value +required "Slack basic auth bearer token") - (@arg SLACKURL: --url +takes_value +required "Base URL of Slack Workspace (e.g. https://[WORKSPACE NAME].slack.com)") - (@arg ALLOWLIST: -a --allowlist +takes_value "Sets a custom allowlist JSON file") - (@arg LATEST: --latest +takes_value "End of time range of messages to include in search") - (@arg OLDEST: --oldest +takes_value "Start of time range of messages to include in search") - ) + let matches: ArgMatches = Command::new("hante_hog") + .version("1.0.11") + .author("Joao Henrique Machado Silva ") + .about("Slack secret scanner in Rust.") + .arg(Arg::new("REGEX").long("regex").action(ArgAction::Set).help("Sets a custom regex JSON file")) + .arg(Arg::new("CHANNELID").long("channelid").required(true).action(ArgAction::Set).help("The ID (e.g. C12345) of the Slack channel you want to scan")) + .arg(Arg::new("VERBOSE").short('v').long("verbose").action(ArgAction::Count).help("Sets the level of debugging information")) + .arg(Arg::new("ENTROPY").long("entropy").action(ArgAction::SetTrue).help("Enables entropy scanning")) + .arg(Arg::new("DEFAULT_ENTROPY_THRESHOLD").long("default_entropy_threshold").action(ArgAction::Set).default_value("0.6").help("Default entropy threshold (0.6 by default)")) + .arg(Arg::new("CASE").long("caseinsensitive").action(ArgAction::SetTrue).help("Sets the case insensitive flag for all regexes")) + .arg(Arg::new("OUTPUT").short('o').long("outputfile").action(ArgAction::Set).help("Sets the path to write the scanner results to (stdout by default)")) + .arg(Arg::new("PRETTYPRINT").long("prettyprint").action(ArgAction::SetTrue).help("Outputs the JSON in human readable format")) + .arg(Arg::new("BEARERTOKEN").long("authtoken").required(true).action(ArgAction::Set).help("Slack basic auth bearer token")) + .arg(Arg::new("SLACKURL").long("url").required(true).action(ArgAction::Set).help("Base URL of Slack Workspace (e.g. https://[WORKSPACE NAME].slack.com)")) + .arg(Arg::new("ALLOWLIST").short('a').long("allowlist").action(ArgAction::Set).help("Sets a custom allowlist JSON file")) + .arg(Arg::new("LATEST").long("latest").action(ArgAction::Set).help("End of time range of messages to include in search")) + .arg(Arg::new("OLDEST").long("oldest").action(ArgAction::Set).help("Start of time range of messages to include in search")) .get_matches(); match run(matches).await { Ok(()) => {} @@ -92,22 +91,24 @@ async fn main() { /// Main logic contained here. Get the CLI variables, create the appropriate TLS objects, /// make the TLS calls, and scan the result.. -async fn run<'b>(arg_matches: ArgMatches<'b>) -> Result<(), SimpleError> { - SecretScanner::set_logging(arg_matches.occurrences_of("VERBOSE")); +async fn run(arg_matches: ArgMatches) -> Result<(), SimpleError> { + SecretScanner::set_logging(arg_matches.get_count("VERBOSE").into()); // initialize the basic variables and CLI options let ssb = SecretScannerBuilder::new().conf_argm(&arg_matches); let secret_scanner = ssb.build(); // Reading the Slack API token from the command line - let slackauthtoken = arg_matches.value_of("BEARERTOKEN"); + let slackauthtoken = arg_matches.get_one::("BEARERTOKEN").map(|s| s.as_str()); // Reading Slack Channel ID from the command line let channel_id = arg_matches - .value_of("CHANNELID") // TODO validate the format somehow + .get_one::("CHANNELID") // TODO validate the format somehow + .map(|s| s.as_str()) .unwrap(); // Reading the Slack URL from the command line let base_url_input = arg_matches - .value_of("SLACKURL") + .get_one::("SLACKURL") + .map(|s| s.as_str()) .unwrap(); // Parse an absolute URL from a string. let base_url_as_url = Url::parse(base_url_input).unwrap(); @@ -115,11 +116,13 @@ async fn run<'b>(arg_matches: ArgMatches<'b>) -> Result<(), SimpleError> { // Reading the latest timestamp from the command line let latest_input = arg_matches - .value_of("LATEST"); + .get_one::("LATEST") + .map(|s| s.as_str()); // Reading the latest timestamp from the command line let oldest_input = arg_matches - .value_of("OLDEST"); + .get_one::("OLDEST") + .map(|s| s.as_str()); // Still inside `async fn main`... let https = hyper_rustls::HttpsConnector::with_native_roots(); From c1000df1226dbc402e0679bd89c3f91fc6ffb360 Mon Sep 17 00:00:00 2001 From: Zach Carlson Date: Sun, 3 Dec 2023 18:31:04 -0600 Subject: [PATCH 06/26] rusty-hogs dep update: clap: ankamali_hog, essex_hog, gottingen_hog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Also remove the #[macro_use] before clap because we no longer use the clap_app! macro, because it no longer exists. ¯\_(ツ)_/¯ --- crates/rusty-hog-scanner/src/lib.rs | 1 - src/bin/ankamali_hog.rs | 46 ++++++++++++------------- src/bin/berkshire_hog.rs | 1 - src/bin/choctaw_hog.rs | 1 - src/bin/duroc_hog.rs | 1 - src/bin/essex_hog.rs | 53 ++++++++++++++--------------- src/bin/gottingen_hog.rs | 53 ++++++++++++++--------------- src/bin/hante_hog.rs | 1 - 8 files changed, 75 insertions(+), 82 deletions(-) diff --git a/crates/rusty-hog-scanner/src/lib.rs b/crates/rusty-hog-scanner/src/lib.rs index e24f1f5..45989fb 100644 --- a/crates/rusty-hog-scanner/src/lib.rs +++ b/crates/rusty-hog-scanner/src/lib.rs @@ -65,7 +65,6 @@ //! assert_eq!(secrets.pop().unwrap(), "Email address"); //! ``` -//#[macro_use] extern crate clap; use anyhow::Result; diff --git a/src/bin/ankamali_hog.rs b/src/bin/ankamali_hog.rs index 1ed796b..5c05059 100644 --- a/src/bin/ankamali_hog.rs +++ b/src/bin/ankamali_hog.rs @@ -23,14 +23,13 @@ //! The ID of the google drive file you want to scan //! ``` -#[macro_use] extern crate clap; extern crate google_drive3 as drive3; extern crate hyper; extern crate hyper_rustls; extern crate yup_oauth2 as oauth2; -use clap::ArgMatches; +use clap::{Arg, ArgAction, ArgMatches, Command}; use drive3::DriveHub; use log::{self, error, info}; use simple_error::SimpleError; @@ -42,22 +41,21 @@ use oauth2::{InstalledFlowAuthenticator, InstalledFlowReturnMethod}; /// Main entry function that uses the [clap crate](https://docs.rs/clap/2.33.0/clap/) #[tokio::main] async fn main() { - let matches = clap_app!(ankamali_hog => - (version: "1.0.11") - (author: "Scott Cutler ") - (about: "Google Drive secret scanner in Rust.") - (@arg REGEX: --regex +takes_value "Sets a custom regex JSON file") - (@arg GDRIVEID: +required "The ID of the Google drive file you want to scan") - (@arg VERBOSE: -v --verbose ... "Sets the level of debugging information") - (@arg ENTROPY: --entropy ... "Enables entropy scanning") - (@arg DEFAULT_ENTROPY_THRESHOLD: --default_entropy_threshold +takes_value "Default entropy threshold (0.6 by default)") - (@arg CASE: --caseinsensitive "Sets the case insensitive flag for all regexes") - (@arg OUTPUT: -o --outputfile +takes_value "Sets the path to write the scanner results to (stdout by default)") - (@arg PRETTYPRINT: --prettyprint "Outputs the JSON in human readable format") - (@arg OAUTHSECRETFILE: --oauthsecret "Path to an OAuth secret file (JSON) ./clientsecret.json by default") - (@arg OAUTHTOKENFILE: --oauthtoken "Path to an OAuth token storage file ./temp_token by default") - (@arg ALLOWLIST: -a --allowlist +takes_value "Sets a custom allowlist JSON file") - ) + let matches = Command::new("ankamali_hog") + .version("1.0.11") + .author("Scott Cutler ") + .about("Google Drive secret scanner in Rust.") + .arg(Arg::new("REGEX").long("regex").action(ArgAction::Set).help("Sets a custom regex JSON file")) + .arg(Arg::new("GDRIVEID").required(true).action(ArgAction::Set).help("The ID of the Google drive file you want to scan")) + .arg(Arg::new("VERBOSE").short('v').long("verbose").action(ArgAction::Count).help("Sets the level of debugging information")) + .arg(Arg::new("ENTROPY").long("entropy").action(ArgAction::SetTrue).help("Enables entropy scanning")) + .arg(Arg::new("DEFAULT_ENTROPY_THRESHOLD").long("default_entropy_threshold").action(ArgAction::Set).help("Default entropy threshold (0.6 by default)")) + .arg(Arg::new("CASE").long("caseinsensitive").action(ArgAction::SetTrue).help("Sets the case insensitive flag for all regexes")) + .arg(Arg::new("OUTPUT").short('o').long("outputfile").action(ArgAction::Set).help("Sets the path to write the scanner results to (stdout by default)")) + .arg(Arg::new("PRETTYPRINT").long("prettyprint").action(ArgAction::SetTrue).help("Outputs the JSON in human readable format")) + .arg(Arg::new("OAUTHSECRETFILE").long("oauthsecret").action(ArgAction::Set).default_value("./clientsecret.json").help("Path to an OAuth secret file (JSON) ./clientsecret.json by default")) + .arg(Arg::new("OAUTHTOKENFILE").long("oauthtoken").action(ArgAction::Set).default_value("./temp_token").help("Path to an OAuth token storage file ./temp_token by default")) + .arg(Arg::new("ALLOWLIST").short('a').long("allowlist").action(ArgAction::Set).help("Sets a custom allowlist JSON file")) .get_matches(); match run(matches).await { Ok(()) => {} @@ -67,17 +65,19 @@ async fn main() { /// Main logic contained here. Get the CLI variables, setup OAuth, setup GDriveScanner and output /// the results. -async fn run<'b>(arg_matches: ArgMatches<'b>) -> Result<(), SimpleError> { // Set logging - SecretScanner::set_logging(arg_matches.occurrences_of("VERBOSE")); +async fn run(arg_matches: ArgMatches) -> Result<(), SimpleError> { // Set logging + SecretScanner::set_logging(arg_matches.get_count("VERBOSE").into()); // Initialize some variables let oauthsecretfile = arg_matches - .value_of("OAUTHSECRETFILE") + .get_one::("OAUTHSECRETFILE") + .map(|s| s.as_str()) .unwrap_or("clientsecret.json"); let oauthtokenfile = arg_matches - .value_of("OAUTHTOKENFILE") + .get_one::("OAUTHTOKENFILE") + .map(|s| s.as_str()) .unwrap_or("temp_token"); - let file_id = arg_matches.value_of("GDRIVEID").unwrap(); + let file_id = arg_matches.get_one::("GDRIVEID").unwrap(); let secret_scanner = SecretScannerBuilder::new().conf_argm(&arg_matches).build(); let gdrive_scanner = GDriveScanner::new_from_scanner(secret_scanner); diff --git a/src/bin/berkshire_hog.rs b/src/bin/berkshire_hog.rs index e0c0ad1..3b3cb9b 100644 --- a/src/bin/berkshire_hog.rs +++ b/src/bin/berkshire_hog.rs @@ -25,7 +25,6 @@ //! Sets the region of the S3 bucket to scan. //! ``` -#[macro_use] extern crate clap; use clap::{Arg, ArgAction, ArgMatches, Command}; diff --git a/src/bin/choctaw_hog.rs b/src/bin/choctaw_hog.rs index 116fccc..3fe2f69 100644 --- a/src/bin/choctaw_hog.rs +++ b/src/bin/choctaw_hog.rs @@ -31,7 +31,6 @@ //! Sets the path (or URL) of the Git repo to scan. SSH links must include username (git@) //! ``` -#[macro_use] extern crate clap; extern crate tempdir; diff --git a/src/bin/duroc_hog.rs b/src/bin/duroc_hog.rs index 38e0b52..255061c 100644 --- a/src/bin/duroc_hog.rs +++ b/src/bin/duroc_hog.rs @@ -25,7 +25,6 @@ //! Sets the path of the file system to scan. //! ``` -#[macro_use] extern crate clap; extern crate tempdir; diff --git a/src/bin/essex_hog.rs b/src/bin/essex_hog.rs index a59d196..a8024d0 100644 --- a/src/bin/essex_hog.rs +++ b/src/bin/essex_hog.rs @@ -29,12 +29,11 @@ //! Example with context: http://example.com:8080/confluence/rest/api/space/ds //! Example without context: http://confluence.myhost.com:8095/rest/api/space/ds -#[macro_use] extern crate clap; extern crate hyper; extern crate hyper_rustls; -use clap::ArgMatches; +use clap::{Arg, ArgAction, ArgMatches, Command}; use encoding::all::ASCII; use encoding::types::Encoding; use encoding::DecoderTrap; @@ -73,24 +72,23 @@ pub struct ConfluencePage { /// Main entry function that uses the [clap crate](https://docs.rs/clap/2.33.0/clap/) #[tokio::main] async fn main() { - let matches = clap_app!(gottingen_hog => - (version: "1.0.11") - (author: "Emily Cain , Scott Cutler") - (about: "Confluence secret scanner in Rust.") - (@arg REGEX: --regex +takes_value "Sets a custom regex JSON file") - (@arg PAGEID: +required "The ID (e.g. 1234) of the confluence page you want to scan") - (@arg URL: +required +takes_value "Base URL of Confluence instance (e.g. https://newrelic.atlassian.net/)") - (@arg VERBOSE: -v --verbose ... "Sets the level of debugging information") - (@arg ENTROPY: --entropy ... "Enables entropy scanning") - (@arg DEFAULT_ENTROPY_THRESHOLD: --default_entropy_threshold +takes_value "Default entropy threshold (0.6 by default)") - (@arg CASE: --caseinsensitive "Sets the case insensitive flag for all regexes") - (@arg OUTPUT: -o --outputfile +takes_value "Sets the path to write the scanner results to (stdout by default)") - (@arg PRETTYPRINT: --prettyprint "Outputs the JSON in human readable format") - (@arg USERNAME: --username +takes_value conflicts_with[AUTHTOKEN] "Confluence username (crafts basic auth header)") - (@arg PASSWORD: --password +takes_value conflicts_with[AUTHTOKEN] "Confluence password (crafts basic auth header)") - (@arg BEARERTOKEN: --authtoken +takes_value conflicts_with[USERNAME PASSWORD] "Confluence basic auth bearer token (instead of user & pass)") - (@arg ALLOWLIST: -a --allowlist +takes_value "Sets a custom allowlist JSON file") - ) + let matches = Command::new("gottingen_hog") + .version("1.0.11") + .author("Emily Cain , Scott Cutler") + .about("Confluence secret scanner in Rust.") + .arg(Arg::new("REGEX").long("regex").action(ArgAction::Set).help("Sets a custom regex JSON file")) + .arg(Arg::new("PAGEID").required(true).action(ArgAction::Set).help("The ID (e.g. 1234) of the confluence page you want to scan")) + .arg(Arg::new("URL").required(true).action(ArgAction::Set).help("Base URL of Confluence instance (e.g. https://newrelic.atlassian.net/)")) + .arg(Arg::new("VERBOSE").short('v').long("verbose").action(ArgAction::Count).help("Sets the level of debugging information")) + .arg(Arg::new("ENTROPY").long("entropy").action(ArgAction::Set).help("Enables entropy scanning")) + .arg(Arg::new("DEFAULT_ENTROPY_THRESHOLD").long("default_entropy_threshold").action(ArgAction::Set).default_value("0.6").help("Default entropy threshold (0.6 by default)")) + .arg(Arg::new("CASE").long("caseinsensitive").action(ArgAction::SetTrue).help("Sets the case insensitive flag for all regexes")) + .arg(Arg::new("OUTPUT").short('o').long("outputfile").action(ArgAction::SetTrue).help("Sets the path to write the scanner results to (stdout by default)")) + .arg(Arg::new("PRETTYPRINT").long("prettyprint").action(ArgAction::SetTrue).help("Outputs the JSON in human readable format")) + .arg(Arg::new("USERNAME").long("username").action(ArgAction::Set).conflicts_with("BEARERTOKEN").help("Confluence username (crafts basic auth header)")) + .arg(Arg::new("PASSWORD").long("password").action(ArgAction::Set).conflicts_with("BEARERTOKEN").help("Confluence password (crafts basic auth header)")) + .arg(Arg::new("BEARERTOKEN").long("authtoken").action(ArgAction::Set).conflicts_with_all(["USERNAME","PASSWORD"]).help("Confluence basic auth bearer token (instead of user & pass)")) + .arg(Arg::new("ALLOWLIST").short('a').long("allowlist").action(ArgAction::Set).help("Sets a custom allowlist JSON file")) .get_matches(); match run(matches).await { Ok(()) => {} @@ -100,23 +98,24 @@ async fn main() { /// Main logic contained here. Get the CLI variables, create the appropriate TLS objects, /// make the TLS calls, and scan the result.. -async fn run<'b>(arg_matches: ArgMatches<'b>) -> Result<(), SimpleError> { - SecretScanner::set_logging(arg_matches.occurrences_of("VERBOSE")); +async fn run(arg_matches: ArgMatches) -> Result<(), SimpleError> { + SecretScanner::set_logging(arg_matches.get_count("VERBOSE").into()); // initialize the basic variables and CLI options let ssb = SecretScannerBuilder::new().conf_argm(&arg_matches); let secret_scanner = ssb.build(); - let jirausername = arg_matches.value_of("USERNAME"); - let jirapassword = arg_matches.value_of("PASSWORD"); - let jiraauthtoken = arg_matches.value_of("BEARERTOKEN"); + let jirausername = arg_matches.get_one::("USERNAME"); + let jirapassword = arg_matches.get_one::("PASSWORD"); + let jiraauthtoken = arg_matches.get_one::("BEARERTOKEN"); let base_url_input = arg_matches - .value_of("URL") + .get_one::("URL") + .map(|s| s.as_str()) .unwrap_or("https://confluence.atlassian.com") .trim_end_matches('/'); let base_url_as_url = Url::parse(base_url_input).unwrap(); let page_id = arg_matches - .value_of("PAGEID") // TODO validate the format somehow + .get_one::("PAGEID") // TODO validate the format somehow .unwrap(); let base_url = base_url_as_url.as_str(); diff --git a/src/bin/gottingen_hog.rs b/src/bin/gottingen_hog.rs index 6de7d8b..1617b74 100644 --- a/src/bin/gottingen_hog.rs +++ b/src/bin/gottingen_hog.rs @@ -22,12 +22,11 @@ //! ARGS: //! The ID (e.g. PROJECT-123) of the Jira issue you want to scan -#[macro_use] extern crate clap; extern crate hyper; extern crate hyper_rustls; -use clap::ArgMatches; +use clap::{Arg, ArgAction, ArgMatches, Command}; use encoding::all::ASCII; use encoding::types::Encoding; use encoding::DecoderTrap; @@ -59,24 +58,23 @@ pub struct JiraFinding { /// Main entry function that uses the [clap crate](https://docs.rs/clap/2.33.0/clap/) #[tokio::main] async fn main() { - let matches: ArgMatches = clap_app!(gottingen_hog => - (version: "1.0.11") - (author: "Emily Cain ") - (about: "Jira secret scanner in Rust.") - (@arg REGEX: --regex +takes_value "Sets a custom regex JSON file") - (@arg JIRAID: +required "The ID (e.g. PROJECT-123) of the Jira issue you want to scan") - (@arg VERBOSE: -v --verbose ... "Sets the level of debugging information") - (@arg ENTROPY: --entropy ... "Enables entropy scanning") - (@arg DEFAULT_ENTROPY_THRESHOLD: --default_entropy_threshold +takes_value "Default entropy threshold (0.6 by default)") - (@arg CASE: --caseinsensitive "Sets the case insensitive flag for all regexes") - (@arg OUTPUT: -o --outputfile +takes_value "Sets the path to write the scanner results to (stdout by default)") - (@arg PRETTYPRINT: --prettyprint "Outputs the JSON in human readable format") - (@arg USERNAME: --username +takes_value conflicts_with[AUTHTOKEN] "Jira username (crafts basic auth header)") - (@arg PASSWORD: --password +takes_value conflicts_with[AUTHTOKEN] "Jira password (crafts basic auth header)") - (@arg BEARERTOKEN: --authtoken +takes_value conflicts_with[USERNAME PASSWORD] "Jira basic auth bearer token (instead of user & pass)") - (@arg JIRAURL: --url +takes_value "Base URL of JIRA instance (e.g. https://jira.atlassian.net/)") - (@arg ALLOWLIST: -a --allowlist +takes_value "Sets a custom allowlist JSON file") - ) + let matches: ArgMatches = Command::new("gottingen_hog") + .version("1.0.11") + .author("Emily Cain ") + .about("Jira secret scanner in Rust.") + .arg(Arg::new("REGEX").long("regex").action(ArgAction::Set).help("Sets a custom regex JSON file")) + .arg(Arg::new("JIRAID").required(true).action(ArgAction::Set).help("The ID (e.g. PROJECT-123) of the Jira issue you want to scan")) + .arg(Arg::new("VERBOSE").short('v').long("verbose").action(ArgAction::Count).help("Sets the level of debugging information")) + .arg(Arg::new("ENTROPY").long("entropy").action(ArgAction::SetTrue).help("Enables entropy scanning")) + .arg(Arg::new("DEFAULT_ENTROPY_THRESHOLD").long("default_entropy_threshold").action(ArgAction::Set).default_value("0.6").help("Default entropy threshold (0.6 by default)")) + .arg(Arg::new("CASE").long("caseinsensitive").action(ArgAction::SetTrue).help("Sets the case insensitive flag for all regexes")) + .arg(Arg::new("OUTPUT").short('o').long("outputfile").action(ArgAction::Set).help("Sets the path to write the scanner results to (stdout by default)")) + .arg(Arg::new("PRETTYPRINT").long("prettyprint").action(ArgAction::SetTrue).help("Outputs the JSON in human readable format")) + .arg(Arg::new("USERNAME").long("username").action(ArgAction::Set).conflicts_with("BEARERTOKEN").help("Jira username (crafts basic auth header)")) + .arg(Arg::new("PASSWORD").long("password").action(ArgAction::Set).conflicts_with("BEARERTOKEN").help("Jira password (crafts basic auth header)")) + .arg(Arg::new("BEARERTOKEN").long("authtoken").action(ArgAction::Set).conflicts_with_all(["USERNAME","PASSWORD"]).help("Jira basic auth bearer token (instead of user & pass)")) + .arg(Arg::new("JIRAURL").long("url").action(ArgAction::Set).help("Base URL of JIRA instance (e.g. https://jira.atlassian.net/)")) + .arg(Arg::new("ALLOWLIST").short('a').long("allowlist").action(ArgAction::Set).help("Sets a custom allowlist JSON file")) .get_matches(); match run(matches).await { Ok(()) => {} @@ -86,22 +84,23 @@ async fn main() { /// Main logic contained here. Get the CLI variables, create the appropriate TLS objects, /// make the TLS calls, and scan the result.. -async fn run<'b>(arg_matches: ArgMatches<'b>) -> Result<(), SimpleError> { - SecretScanner::set_logging(arg_matches.occurrences_of("VERBOSE")); +async fn run(arg_matches: ArgMatches) -> Result<(), SimpleError> { + SecretScanner::set_logging(arg_matches.get_count("VERBOSE").into()); // initialize the basic variables and CLI options let ssb = SecretScannerBuilder::new().conf_argm(&arg_matches); let secret_scanner = ssb.build(); - let jirausername = arg_matches.value_of("USERNAME"); - let jirapassword = arg_matches.value_of("PASSWORD"); - let jiraauthtoken = arg_matches.value_of("BEARERTOKEN"); + let jirausername = arg_matches.get_one::("USERNAME"); + let jirapassword = arg_matches.get_one::("PASSWORD"); + let jiraauthtoken = arg_matches.get_one::("BEARERTOKEN"); let base_url_input = arg_matches - .value_of("JIRAURL") + .get_one::("JIRAURL") + .map(|s| s.as_str()) .unwrap_or("https://jira.atlassian.com/"); let base_url_as_url = Url::parse(base_url_input).unwrap(); let issue_id = arg_matches - .value_of("JIRAID") // TODO validate the format somehow + .get_one::("JIRAID") // TODO validate the format somehow .unwrap(); let base_url = base_url_as_url.as_str(); diff --git a/src/bin/hante_hog.rs b/src/bin/hante_hog.rs index a804fe0..df00852 100644 --- a/src/bin/hante_hog.rs +++ b/src/bin/hante_hog.rs @@ -27,7 +27,6 @@ //! --url //! Base URL of Slack Workspace (e.g. https://[WORKSPACE NAME].slack.com) -#[macro_use] extern crate clap; extern crate hyper; extern crate hyper_rustls; From 8d83222a5cda713a9d3364efd9ef9311556e5231 Mon Sep 17 00:00:00 2001 From: Zach Carlson Date: Sun, 3 Dec 2023 18:38:12 -0600 Subject: [PATCH 07/26] Update some deps I know haven't changed --- Cargo.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c21bb96..e88a5e8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,7 @@ license = "Apache-2.0" [dependencies] rusty_hog_scanner = { path = "crates/rusty-hog-scanner" } tokio = { version = "1", features = ["full"] } -git2 = "0.13" +git2 = "0.18" serde = "1.0" serde_json = "1.0" serde_derive = "^1" @@ -28,8 +28,8 @@ url = "2" tempdir = "0.3" base64 = "0.13" log = "0.4" -simple_logger = "1.11" -simple-error = "0.2" +simple_logger = "4.3" +simple-error = "0.3" chrono = "0.4" encoding = "0.2" hex = "0.4" From 60fb41c337ba48f7954908d1e8fef5f0fe0d379c Mon Sep 17 00:00:00 2001 From: Zach Carlson Date: Sun, 3 Dec 2023 18:39:55 -0600 Subject: [PATCH 08/26] Couple more bumps that worked out --- Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e88a5e8..200d917 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,11 +40,11 @@ hyper = "^0.14" hyper-rustls = "^0.22" yup-oauth2 = "^5.0" walkdir = "2" -zip = "0.5" +zip = "0.6" tar = "0.4" flate2 = "1.0" tempfile = "3.2" -path-clean = "0.1.0" +path-clean = "1.0" anyhow = "1.0" [dev-dependencies] From 21835ffb8617d15fed42883e144f50d8186dc30e Mon Sep 17 00:00:00 2001 From: Zach Carlson Date: Sun, 3 Dec 2023 18:42:38 -0600 Subject: [PATCH 09/26] base64 --- Cargo.toml | 2 +- src/bin/essex_hog.rs | 3 ++- src/bin/gottingen_hog.rs | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 200d917..5267e46 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,7 +26,7 @@ clap = "4" regex = "1" url = "2" tempdir = "0.3" -base64 = "0.13" +base64 = "0.21" log = "0.4" simple_logger = "4.3" simple-error = "0.3" diff --git a/src/bin/essex_hog.rs b/src/bin/essex_hog.rs index a8024d0..5689ba2 100644 --- a/src/bin/essex_hog.rs +++ b/src/bin/essex_hog.rs @@ -33,6 +33,7 @@ extern crate clap; extern crate hyper; extern crate hyper_rustls; +use base64::{Engine as _, engine::general_purpose as Base64Engine}; use clap::{Arg, ArgAction, ArgMatches, Command}; use encoding::all::ASCII; use encoding::types::Encoding; @@ -130,7 +131,7 @@ async fn run(arg_matches: ArgMatches) -> Result<(), SimpleError> { Some(u) => { format!( "Basic {}", - base64::encode(format!("{}:{}", u, jirapassword.unwrap())) + Base64Engine::STANDARD_NO_PAD.encode(format!("{}:{}", u, jirapassword.unwrap())) ) } // otherwise use AUTHTOKEN to craft the auth header diff --git a/src/bin/gottingen_hog.rs b/src/bin/gottingen_hog.rs index 1617b74..31b2259 100644 --- a/src/bin/gottingen_hog.rs +++ b/src/bin/gottingen_hog.rs @@ -26,6 +26,7 @@ extern crate clap; extern crate hyper; extern crate hyper_rustls; +use base64::{Engine as _, engine::general_purpose as Base64Engine}; use clap::{Arg, ArgAction, ArgMatches, Command}; use encoding::all::ASCII; use encoding::types::Encoding; @@ -115,7 +116,7 @@ async fn run(arg_matches: ArgMatches) -> Result<(), SimpleError> { Some(u) => { format!( "Basic {}", - base64::encode(format!("{}:{}", u, jirapassword.unwrap())) + Base64Engine::STANDARD_NO_PAD.encode(format!("{}:{}", u, jirapassword.unwrap())) ) } // otherwise use AUTHTOKEN to craft the auth header From 9a5f58002cc8c24cb97fb0f884bdc8bdd1a20c12 Mon Sep 17 00:00:00 2001 From: Zach Carlson Date: Sun, 3 Dec 2023 18:44:22 -0600 Subject: [PATCH 10/26] lambda_runtime --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 5267e46..5f3817e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,7 +33,7 @@ simple-error = "0.3" chrono = "0.4" encoding = "0.2" hex = "0.4" -lambda_runtime = "0.3" +lambda_runtime = "0.8" rust-s3 = "0.26" google-drive3 = "2.0.4" hyper = "^0.14" From c1d3a5cf16890dc45e1312f146eb03142eb0491a Mon Sep 17 00:00:00 2001 From: Zach Carlson Date: Sun, 3 Dec 2023 18:55:48 -0600 Subject: [PATCH 11/26] rust-s3 --- Cargo.toml | 4 ++-- src/aws_scanning.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5f3817e..5276576 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,7 +34,7 @@ chrono = "0.4" encoding = "0.2" hex = "0.4" lambda_runtime = "0.8" -rust-s3 = "0.26" +rust-s3 = { version = "0.33", features = ["blocking"] } google-drive3 = "2.0.4" hyper = "^0.14" hyper-rustls = "^0.22" @@ -52,4 +52,4 @@ escargot = "0.5.0" [profile.release] lto = true -codegen-units = 1 \ No newline at end of file +codegen-units = 1 diff --git a/src/aws_scanning.rs b/src/aws_scanning.rs index 8acc035..52e0149 100644 --- a/src/aws_scanning.rs +++ b/src/aws_scanning.rs @@ -105,8 +105,8 @@ impl S3Scanner { let mut output: Vec = Vec::new(); // Get the actual data from S3 - let (data, code) = match bucket.get_object_blocking(filepath) { - Ok(x) => (x.0, x.1), + let (code, data) = match bucket.get_object_blocking(filepath) { + Ok(x) => (x.status_code(), x.to_vec()), Err(e) => return Err(SimpleError::new(e.to_string())), }; trace!("Code: {}\nData: {:?}", code, data); From 401d39f2fb4c2785194b4ccecb5d98d3a268168a Mon Sep 17 00:00:00 2001 From: Zach Carlson Date: Sun, 3 Dec 2023 19:02:57 -0600 Subject: [PATCH 12/26] rust-s3 again ListBucketResult changed slightly --- src/bin/berkshire_hog.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bin/berkshire_hog.rs b/src/bin/berkshire_hog.rs index 3b3cb9b..f479878 100644 --- a/src/bin/berkshire_hog.rs +++ b/src/bin/berkshire_hog.rs @@ -131,7 +131,7 @@ fn run(arg_matches: &ArgMatches) -> Result<(), SimpleError> { }; let mut keys: Vec = results .into_iter() - .flat_map(|x| x.0.contents) + .flat_map(|x| x.contents) .map(|x| x.key) .filter(|x| !x.ends_with('/')) .collect(); From f6b748fc544c81f66bfc500f7e4d5e858a1cb295 Mon Sep 17 00:00:00 2001 From: Zach Carlson Date: Sun, 3 Dec 2023 22:29:05 -0600 Subject: [PATCH 13/26] Update hyper_rustls, hyper, and google-drive3 Unfortunately google-drive3 really requires hyper <1.0 so we're stuck on 0.14. Also I'm committing this here because I kind of tied to a specific connector and I don't really want to do that long-term, but I also don't want to lose the functional state I have it in right now :D --- src/bin/ankamali_hog.rs | 9 ++++++++- src/bin/essex_hog.rs | 6 +++++- src/bin/gottingen_hog.rs | 6 +++++- src/bin/hante_hog.rs | 6 +++++- src/google_scanning.rs | 11 ++++++----- 5 files changed, 29 insertions(+), 9 deletions(-) diff --git a/src/bin/ankamali_hog.rs b/src/bin/ankamali_hog.rs index 5c05059..45e535f 100644 --- a/src/bin/ankamali_hog.rs +++ b/src/bin/ankamali_hog.rs @@ -90,7 +90,14 @@ async fn run(arg_matches: ArgMatches) -> Result<(), SimpleError> { // Set log .build() .await .expect("failed to create authenticator (try deleting temp_token and restarting)"); - let hub = DriveHub::new(hyper::Client::builder().build(hyper_rustls::HttpsConnector::with_native_roots()), auth); + let hub = DriveHub::new( + hyper::Client::builder().build( + hyper_rustls::HttpsConnectorBuilder::new() + .with_native_roots() + .https_only() + .enable_all_versions() + .build() + ), auth); // get some initial info about the file let gdriveinfo = GDriveFileInfo::new(file_id, &hub).await.unwrap(); diff --git a/src/bin/essex_hog.rs b/src/bin/essex_hog.rs index 5689ba2..86ace11 100644 --- a/src/bin/essex_hog.rs +++ b/src/bin/essex_hog.rs @@ -122,7 +122,11 @@ async fn run(arg_matches: ArgMatches) -> Result<(), SimpleError> { let base_url = base_url_as_url.as_str(); // Still inside `async fn main`... - let https = hyper_rustls::HttpsConnector::with_native_roots(); + let https = hyper_rustls::HttpsConnectorBuilder::new() + .with_native_roots() + .https_only() + .enable_all_versions() + .build(); let hyper_client: client::Client<_, hyper::Body> = client::Client::builder().build(https); // TODO: Support other modes of JIRA authentication diff --git a/src/bin/gottingen_hog.rs b/src/bin/gottingen_hog.rs index 31b2259..be61d32 100644 --- a/src/bin/gottingen_hog.rs +++ b/src/bin/gottingen_hog.rs @@ -107,7 +107,11 @@ async fn run(arg_matches: ArgMatches) -> Result<(), SimpleError> { let base_url = base_url_as_url.as_str(); // Still inside `async fn main`... - let https = hyper_rustls::HttpsConnector::with_native_roots(); + let https = hyper_rustls::HttpsConnectorBuilder::new() + .with_native_roots() + .https_only() + .enable_all_versions() + .build(); let hyper_client: client::Client<_, hyper::Body> = client::Client::builder().build(https); // TODO: Support other modes of JIRA authentication diff --git a/src/bin/hante_hog.rs b/src/bin/hante_hog.rs index df00852..7983089 100644 --- a/src/bin/hante_hog.rs +++ b/src/bin/hante_hog.rs @@ -124,7 +124,11 @@ async fn run(arg_matches: ArgMatches) -> Result<(), SimpleError> { .map(|s| s.as_str()); // Still inside `async fn main`... - let https = hyper_rustls::HttpsConnector::with_native_roots(); + let https = hyper_rustls::HttpsConnectorBuilder::new() + .with_native_roots() + .https_only() + .enable_all_versions() + .build(); let hyper_client: client::Client<_, hyper::Body> = client::Client::builder().build(https); // Construction Authentication header diff --git a/src/google_scanning.rs b/src/google_scanning.rs index e33d248..f26d4bb 100644 --- a/src/google_scanning.rs +++ b/src/google_scanning.rs @@ -75,6 +75,7 @@ extern crate google_drive3 as drive3; extern crate yup_oauth2 as oauth2; +use chrono::{Utc, DateTime}; use drive3::DriveHub; use encoding::all::ASCII; use encoding::{DecoderTrap, Encoding}; @@ -101,7 +102,7 @@ use rusty_hog_scanner::SecretScanner; /// }; /// ``` pub struct GDriveFinding { - pub date: String, + pub date: DateTime, pub diff: String, pub path: String, #[serde(rename = "stringsFound")] @@ -140,7 +141,7 @@ pub struct GDriveScanner { pub struct GDriveFileInfo { pub file_id: String, pub mime_type: String, - pub modified_time: String, + pub modified_time: DateTime, pub web_link: String, pub parents: Vec, pub name: String, @@ -149,7 +150,7 @@ pub struct GDriveFileInfo { impl GDriveFileInfo { /// Construct a `GDriveFileInfo` object from a Google Drive File ID and an authorized `DriveHub` object - pub async fn new(file_id: &str, hub: &DriveHub) -> Result { + pub async fn new(file_id: &str, hub: &DriveHub>) -> Result { let fields = "kind, id, name, mimeType, webViewLink, modifiedTime, parents"; let hub_result = hub .files() @@ -211,7 +212,7 @@ impl GDriveScanner { /// Google Drive. Expect authorization issues here if you don't have access to the file. async fn gdrive_file_contents( gdrivefile: &GDriveFileInfo, - hub: &DriveHub, + hub: &DriveHub>, ) -> Result, SimpleError> { let resp_obj = hub .files() @@ -232,7 +233,7 @@ impl GDriveScanner { pub async fn perform_scan( &self, gdrivefile: &GDriveFileInfo, - hub: &DriveHub, + hub: &DriveHub>, ) -> HashSet { // download an export of the file, split on new lines, store in lines let buffer = Self::gdrive_file_contents(gdrivefile, hub).await.unwrap(); From a22167065cc2ba88a81ae435ad2bb7e109929187 Mon Sep 17 00:00:00 2001 From: Zach Carlson Date: Mon, 4 Dec 2023 05:09:47 -0600 Subject: [PATCH 14/26] Do a more appropriately generic structure So this was unfortunate: at some point DriveHub became a DriveHub. Fine, carry the S through, right? Unfortunately some of the downstream calls (specifically anything after the "activity" e.g. files()) require specific trait bounds -- which means everything you pass the DriveHub to has to enforce those trait bounds. This *would* be fine if the DriveHub were just a *member* of the struct: we could specify S and its bounds once and not have to worry about it, but that would require a more thorough rewrite, which I can't fully test without access to certain things. So I settled for using the trait bounds required downstream as well everywhere, which adds about five lines everywhere, but at least required very little further modification to get the same functionality. --- src/google_scanning.rs | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/src/google_scanning.rs b/src/google_scanning.rs index f26d4bb..4630406 100644 --- a/src/google_scanning.rs +++ b/src/google_scanning.rs @@ -84,6 +84,8 @@ use hyper::body; use serde_derive::{Deserialize, Serialize}; use simple_error::SimpleError; use std::collections::HashSet; +use std::error::Error as StdError; +use tokio::io::{AsyncRead, AsyncWrite}; use rusty_hog_scanner::SecretScanner; #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Hash, Clone, Default)] @@ -150,7 +152,13 @@ pub struct GDriveFileInfo { impl GDriveFileInfo { /// Construct a `GDriveFileInfo` object from a Google Drive File ID and an authorized `DriveHub` object - pub async fn new(file_id: &str, hub: &DriveHub>) -> Result { + pub async fn new(file_id: &str, hub: &DriveHub) -> Result + where + S: hyper::service::Service + Clone + Send + Sync + 'static, + S::Response: hyper::client::connect::Connection + AsyncRead + AsyncWrite + Send + Unpin + 'static, + S::Future: Send + Unpin + 'static, + S::Error: Into> + { let fields = "kind, id, name, mimeType, webViewLink, modifiedTime, parents"; let hub_result = hub .files() @@ -210,10 +218,16 @@ impl GDriveScanner { /// Takes information about the file, and the DriveHub object, and retrieves the content from /// Google Drive. Expect authorization issues here if you don't have access to the file. - async fn gdrive_file_contents( + async fn gdrive_file_contents( gdrivefile: &GDriveFileInfo, - hub: &DriveHub>, - ) -> Result, SimpleError> { + hub: &DriveHub, + ) -> Result, SimpleError> + where + S: hyper::service::Service + Clone + Send + Sync + 'static, + S::Response: hyper::client::connect::Connection + AsyncRead + AsyncWrite + Send + Unpin + 'static, + S::Future: Send + Unpin + 'static, + S::Error: Into> + { let resp_obj = hub .files() .export(&gdrivefile.file_id, &gdrivefile.mime_type) @@ -230,11 +244,17 @@ impl GDriveScanner { /// Takes information about the file, and the DriveHub object, and return a list of findings. /// This calls get_file_contents(), so expect an HTTPS call to GDrive. - pub async fn perform_scan( + pub async fn perform_scan( &self, gdrivefile: &GDriveFileInfo, - hub: &DriveHub>, - ) -> HashSet { + hub: &DriveHub, + ) -> HashSet + where + S: hyper::service::Service + Clone + Send + Sync + 'static, + S::Response: hyper::client::connect::Connection + AsyncRead + AsyncWrite + Send + Unpin + 'static, + S::Future: Send + Unpin + 'static, + S::Error: Into> + { // download an export of the file, split on new lines, store in lines let buffer = Self::gdrive_file_contents(gdrivefile, hub).await.unwrap(); let lines = buffer.split(|x| (*x as char) == '\n'); From 86035e1efebf164a843bb178d13a6ecd90d11141 Mon Sep 17 00:00:00 2001 From: Zach Carlson Date: Mon, 4 Dec 2023 05:15:26 -0600 Subject: [PATCH 15/26] rustfmt google_scanning --- src/google_scanning.rs | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/google_scanning.rs b/src/google_scanning.rs index 4630406..578d3e1 100644 --- a/src/google_scanning.rs +++ b/src/google_scanning.rs @@ -75,18 +75,18 @@ extern crate google_drive3 as drive3; extern crate yup_oauth2 as oauth2; -use chrono::{Utc, DateTime}; +use chrono::{DateTime, Utc}; use drive3::DriveHub; use encoding::all::ASCII; use encoding::{DecoderTrap, Encoding}; use google_drive3::api::Scope; use hyper::body; +use rusty_hog_scanner::SecretScanner; use serde_derive::{Deserialize, Serialize}; use simple_error::SimpleError; use std::collections::HashSet; use std::error::Error as StdError; use tokio::io::{AsyncRead, AsyncWrite}; -use rusty_hog_scanner::SecretScanner; #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Hash, Clone, Default)] /// `serde_json` object that represents a single found secret - finding @@ -152,12 +152,13 @@ pub struct GDriveFileInfo { impl GDriveFileInfo { /// Construct a `GDriveFileInfo` object from a Google Drive File ID and an authorized `DriveHub` object - pub async fn new(file_id: &str, hub: &DriveHub) -> Result + pub async fn new(file_id: &str, hub: &DriveHub) -> Result where S: hyper::service::Service + Clone + Send + Sync + 'static, - S::Response: hyper::client::connect::Connection + AsyncRead + AsyncWrite + Send + Unpin + 'static, + S::Response: + hyper::client::connect::Connection + AsyncRead + AsyncWrite + Send + Unpin + 'static, S::Future: Send + Unpin + 'static, - S::Error: Into> + S::Error: Into>, { let fields = "kind, id, name, mimeType, webViewLink, modifiedTime, parents"; let hub_result = hub @@ -224,9 +225,10 @@ impl GDriveScanner { ) -> Result, SimpleError> where S: hyper::service::Service + Clone + Send + Sync + 'static, - S::Response: hyper::client::connect::Connection + AsyncRead + AsyncWrite + Send + Unpin + 'static, + S::Response: + hyper::client::connect::Connection + AsyncRead + AsyncWrite + Send + Unpin + 'static, S::Future: Send + Unpin + 'static, - S::Error: Into> + S::Error: Into>, { let resp_obj = hub .files() @@ -248,12 +250,13 @@ impl GDriveScanner { &self, gdrivefile: &GDriveFileInfo, hub: &DriveHub, - ) -> HashSet + ) -> HashSet where S: hyper::service::Service + Clone + Send + Sync + 'static, - S::Response: hyper::client::connect::Connection + AsyncRead + AsyncWrite + Send + Unpin + 'static, + S::Response: + hyper::client::connect::Connection + AsyncRead + AsyncWrite + Send + Unpin + 'static, S::Future: Send + Unpin + 'static, - S::Error: Into> + S::Error: Into>, { // download an export of the file, split on new lines, store in lines let buffer = Self::gdrive_file_contents(gdrivefile, hub).await.unwrap(); From 48a0d30aa88531c455f9e1179b20f8971acd80ac Mon Sep 17 00:00:00 2001 From: Zach Carlson Date: Mon, 4 Dec 2023 05:16:23 -0600 Subject: [PATCH 16/26] Update deps in Cargo.toml --- Cargo.toml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5276576..19006c7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ members = [ name = "rusty_hogs" version = "1.0.11" authors = ["Scott Cutler "] -edition = "2018" +edition = "2021" description = "This project provides a set of scanners that will use regular expressions to try and detect the presence of sensitive information such as API keys, passwords, and personal information. It includes a set of regular expressions by default, but will also accept a JSON object containing your custom regular expressions." homepage = "https://github.com/newrelic/rusty-hog" keywords = ["secret", "scanner", "regex", "rusty", "hog"] @@ -35,10 +35,10 @@ encoding = "0.2" hex = "0.4" lambda_runtime = "0.8" rust-s3 = { version = "0.33", features = ["blocking"] } -google-drive3 = "2.0.4" -hyper = "^0.14" -hyper-rustls = "^0.22" -yup-oauth2 = "^5.0" +google-drive3 = "5.0" +hyper = { version = "^0.14", features = ["client"] } +hyper-rustls = "^0.24" +yup-oauth2 = "^8.3" walkdir = "2" zip = "0.6" tar = "0.4" From f84a962b9fbfadcc7cb7e82885c2135144694b4e Mon Sep 17 00:00:00 2001 From: Zach Carlson Date: Mon, 4 Dec 2023 05:18:31 -0600 Subject: [PATCH 17/26] rustfmt the rest of the owl --- src/aws_scanning.rs | 2 +- src/bin/ankamali_hog.rs | 103 +++++++++++++++---- src/bin/berkshire_hog.rs | 2 +- src/bin/berkshire_hog_lambda.rs | 2 +- src/bin/choctaw_hog.rs | 31 ++++-- src/bin/duroc_hog.rs | 87 ++++++++++++++-- src/bin/essex_hog.rs | 100 +++++++++++++++--- src/bin/gottingen_hog.rs | 100 +++++++++++++++--- src/bin/hante_hog.rs | 173 +++++++++++++++++++++++++------- src/git_scanning.rs | 2 +- 10 files changed, 492 insertions(+), 110 deletions(-) diff --git a/src/aws_scanning.rs b/src/aws_scanning.rs index 52e0149..c1f214e 100644 --- a/src/aws_scanning.rs +++ b/src/aws_scanning.rs @@ -53,11 +53,11 @@ use encoding::all::ASCII; use encoding::{DecoderTrap, Encoding}; use log::{self, error, trace}; +use rusty_hog_scanner::SecretScanner; use s3::bucket::Bucket; use serde_derive::{Deserialize, Serialize}; use simple_error::SimpleError; use std::str; -use rusty_hog_scanner::SecretScanner; #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Hash, Clone, Default)] /// `serde_json` object that represents a single found secret - finding diff --git a/src/bin/ankamali_hog.rs b/src/bin/ankamali_hog.rs index 45e535f..8e0fd6f 100644 --- a/src/bin/ankamali_hog.rs +++ b/src/bin/ankamali_hog.rs @@ -32,11 +32,11 @@ extern crate yup_oauth2 as oauth2; use clap::{Arg, ArgAction, ArgMatches, Command}; use drive3::DriveHub; use log::{self, error, info}; +use oauth2::{InstalledFlowAuthenticator, InstalledFlowReturnMethod}; +use rusty_hog_scanner::{SecretScanner, SecretScannerBuilder}; +use rusty_hogs::google_scanning::{GDriveFileInfo, GDriveScanner}; use simple_error::SimpleError; use std::path::Path; -use rusty_hogs::google_scanning::{GDriveFileInfo, GDriveScanner}; -use rusty_hog_scanner::{SecretScanner, SecretScannerBuilder}; -use oauth2::{InstalledFlowAuthenticator, InstalledFlowReturnMethod}; /// Main entry function that uses the [clap crate](https://docs.rs/clap/2.33.0/clap/) #[tokio::main] @@ -45,17 +45,77 @@ async fn main() { .version("1.0.11") .author("Scott Cutler ") .about("Google Drive secret scanner in Rust.") - .arg(Arg::new("REGEX").long("regex").action(ArgAction::Set).help("Sets a custom regex JSON file")) - .arg(Arg::new("GDRIVEID").required(true).action(ArgAction::Set).help("The ID of the Google drive file you want to scan")) - .arg(Arg::new("VERBOSE").short('v').long("verbose").action(ArgAction::Count).help("Sets the level of debugging information")) - .arg(Arg::new("ENTROPY").long("entropy").action(ArgAction::SetTrue).help("Enables entropy scanning")) - .arg(Arg::new("DEFAULT_ENTROPY_THRESHOLD").long("default_entropy_threshold").action(ArgAction::Set).help("Default entropy threshold (0.6 by default)")) - .arg(Arg::new("CASE").long("caseinsensitive").action(ArgAction::SetTrue).help("Sets the case insensitive flag for all regexes")) - .arg(Arg::new("OUTPUT").short('o').long("outputfile").action(ArgAction::Set).help("Sets the path to write the scanner results to (stdout by default)")) - .arg(Arg::new("PRETTYPRINT").long("prettyprint").action(ArgAction::SetTrue).help("Outputs the JSON in human readable format")) - .arg(Arg::new("OAUTHSECRETFILE").long("oauthsecret").action(ArgAction::Set).default_value("./clientsecret.json").help("Path to an OAuth secret file (JSON) ./clientsecret.json by default")) - .arg(Arg::new("OAUTHTOKENFILE").long("oauthtoken").action(ArgAction::Set).default_value("./temp_token").help("Path to an OAuth token storage file ./temp_token by default")) - .arg(Arg::new("ALLOWLIST").short('a').long("allowlist").action(ArgAction::Set).help("Sets a custom allowlist JSON file")) + .arg( + Arg::new("REGEX") + .long("regex") + .action(ArgAction::Set) + .help("Sets a custom regex JSON file"), + ) + .arg( + Arg::new("GDRIVEID") + .required(true) + .action(ArgAction::Set) + .help("The ID of the Google drive file you want to scan"), + ) + .arg( + Arg::new("VERBOSE") + .short('v') + .long("verbose") + .action(ArgAction::Count) + .help("Sets the level of debugging information"), + ) + .arg( + Arg::new("ENTROPY") + .long("entropy") + .action(ArgAction::SetTrue) + .help("Enables entropy scanning"), + ) + .arg( + Arg::new("DEFAULT_ENTROPY_THRESHOLD") + .long("default_entropy_threshold") + .action(ArgAction::Set) + .help("Default entropy threshold (0.6 by default)"), + ) + .arg( + Arg::new("CASE") + .long("caseinsensitive") + .action(ArgAction::SetTrue) + .help("Sets the case insensitive flag for all regexes"), + ) + .arg( + Arg::new("OUTPUT") + .short('o') + .long("outputfile") + .action(ArgAction::Set) + .help("Sets the path to write the scanner results to (stdout by default)"), + ) + .arg( + Arg::new("PRETTYPRINT") + .long("prettyprint") + .action(ArgAction::SetTrue) + .help("Outputs the JSON in human readable format"), + ) + .arg( + Arg::new("OAUTHSECRETFILE") + .long("oauthsecret") + .action(ArgAction::Set) + .default_value("./clientsecret.json") + .help("Path to an OAuth secret file (JSON) ./clientsecret.json by default"), + ) + .arg( + Arg::new("OAUTHTOKENFILE") + .long("oauthtoken") + .action(ArgAction::Set) + .default_value("./temp_token") + .help("Path to an OAuth token storage file ./temp_token by default"), + ) + .arg( + Arg::new("ALLOWLIST") + .short('a') + .long("allowlist") + .action(ArgAction::Set) + .help("Sets a custom allowlist JSON file"), + ) .get_matches(); match run(matches).await { Ok(()) => {} @@ -65,7 +125,8 @@ async fn main() { /// Main logic contained here. Get the CLI variables, setup OAuth, setup GDriveScanner and output /// the results. -async fn run(arg_matches: ArgMatches) -> Result<(), SimpleError> { // Set logging +async fn run(arg_matches: ArgMatches) -> Result<(), SimpleError> { + // Set logging SecretScanner::set_logging(arg_matches.get_count("VERBOSE").into()); // Initialize some variables @@ -93,11 +154,13 @@ async fn run(arg_matches: ArgMatches) -> Result<(), SimpleError> { // Set log let hub = DriveHub::new( hyper::Client::builder().build( hyper_rustls::HttpsConnectorBuilder::new() - .with_native_roots() - .https_only() - .enable_all_versions() - .build() - ), auth); + .with_native_roots() + .https_only() + .enable_all_versions() + .build(), + ), + auth, + ); // get some initial info about the file let gdriveinfo = GDriveFileInfo::new(file_id, &hub).await.unwrap(); diff --git a/src/bin/berkshire_hog.rs b/src/bin/berkshire_hog.rs index f479878..d716e2a 100644 --- a/src/bin/berkshire_hog.rs +++ b/src/bin/berkshire_hog.rs @@ -37,8 +37,8 @@ use simple_error::{require_with, try_with}; use std::str; use url::Url; -use rusty_hogs::aws_scanning::{S3Finding, S3Scanner}; use rusty_hog_scanner::{SecretScanner, SecretScannerBuilder}; +use rusty_hogs::aws_scanning::{S3Finding, S3Scanner}; use std::collections::HashSet; /// Main entry function that uses the [clap crate](https://docs.rs/clap/2.33.0/clap/) diff --git a/src/bin/berkshire_hog_lambda.rs b/src/bin/berkshire_hog_lambda.rs index b5c8245..e8cfc50 100644 --- a/src/bin/berkshire_hog_lambda.rs +++ b/src/bin/berkshire_hog_lambda.rs @@ -18,8 +18,8 @@ extern crate s3; use lambda_runtime::{handler_fn, Context, Error}; use log::{self, warn, LevelFilter}; -use rusty_hogs::aws_scanning::{S3Finding, S3Scanner}; use rusty_hog_scanner::SecretScannerBuilder; +use rusty_hogs::aws_scanning::{S3Finding, S3Scanner}; use s3::bucket::Bucket; use s3::creds::Credentials; use s3::region::Region; diff --git a/src/bin/choctaw_hog.rs b/src/bin/choctaw_hog.rs index 3fe2f69..a9ece73 100644 --- a/src/bin/choctaw_hog.rs +++ b/src/bin/choctaw_hog.rs @@ -45,8 +45,8 @@ use simple_error::SimpleError; use std::str; use tempdir::TempDir; -use rusty_hogs::git_scanning::GitScanner; use rusty_hog_scanner::{SecretScanner, SecretScannerBuilder}; +use rusty_hogs::git_scanning::GitScanner; /// Main entry function that uses the [clap crate](https://docs.rs/clap/2.33.0/clap/) fn main() { @@ -84,12 +84,24 @@ fn run(arg_matches: &ArgMatches) -> Result<(), SimpleError> { // Initialize some more variables let secret_scanner = SecretScannerBuilder::new().conf_argm(arg_matches).build(); - let sshkeypath = arg_matches.get_one::("SSHKEYPATH").map(|s| s.as_str()); - let sshkeyphrase = arg_matches.get_one::("SSHKEYPHRASE").map(|s| s.as_str()); - let httpsuser = arg_matches.get_one::("HTTPSUSER").map(|s| s.as_str()); - let httpspass = arg_matches.get_one::("HTTPSPASS").map(|s| s.as_str()); - let since_commit = arg_matches.get_one::("SINCECOMMIT").map(|s| s.as_str()); - let until_commit = arg_matches.get_one::("UNTILCOMMIT").map(|s| s.as_str()); + let sshkeypath = arg_matches + .get_one::("SSHKEYPATH") + .map(|s| s.as_str()); + let sshkeyphrase = arg_matches + .get_one::("SSHKEYPHRASE") + .map(|s| s.as_str()); + let httpsuser = arg_matches + .get_one::("HTTPSUSER") + .map(|s| s.as_str()); + let httpspass = arg_matches + .get_one::("HTTPSPASS") + .map(|s| s.as_str()); + let since_commit = arg_matches + .get_one::("SINCECOMMIT") + .map(|s| s.as_str()); + let until_commit = arg_matches + .get_one::("UNTILCOMMIT") + .map(|s| s.as_str()); let recent_days: Option = match arg_matches.get_one::("RECENTDAYS") { Some(d) => { if *d == 0 { @@ -104,7 +116,10 @@ fn run(arg_matches: &ArgMatches) -> Result<(), SimpleError> { // Get Git objects let dest_dir = TempDir::new("rusty_hogs").unwrap(); let dest_dir_path = dest_dir.path(); - let source_path: &str = arg_matches.get_one::("GITPATH").map(|s| s.as_str()).unwrap(); + let source_path: &str = arg_matches + .get_one::("GITPATH") + .map(|s| s.as_str()) + .unwrap(); // Do the scan let git_scanner = GitScanner::new_from_scanner(secret_scanner).init_git_repo( diff --git a/src/bin/duroc_hog.rs b/src/bin/duroc_hog.rs index 255061c..352bca6 100644 --- a/src/bin/duroc_hog.rs +++ b/src/bin/duroc_hog.rs @@ -71,17 +71,82 @@ fn main() { .version("1.0.11") .author("Scott Cutler ") .about("File system secret scanner in Rust") - .arg(Arg::new("REGEX").short('r').long("regex").action(ArgAction::Set).value_name("REGEX").help("Sets a custom regex JSON file")) - .arg(Arg::new("FSPATH").required(true).action(ArgAction::Set).value_name("PATH").help("Sets the path of the directory or file to scan.")) - .arg(Arg::new("NORECURSIVE").long("norecursive").action(ArgAction::SetTrue).help("Disable recursive scanning of all subdirectories underneath the supplied path")) - .arg(Arg::new("VERBOSE").short('v').long("verbose").action(ArgAction::Count).help("Sets the level of debugging information")) - .arg(Arg::new("ENTROPY").long("entropy").action(ArgAction::SetTrue).help("Enables entropy scanning")) - .arg(Arg::new("DEFAULT_ENTROPY_THRESHOLD").long("default_entropy_threshold").action(ArgAction::Set).default_value("0.6").help("Default entropy threshold (0.6 by default)")) - .arg(Arg::new("UNZIP").short('z').long("unzip").action(ArgAction::SetTrue).help("Recursively scans archives (ZIP and TAR) in memory (dangerous)")) - .arg(Arg::new("CASE").long("caseinsensitive").action(ArgAction::SetTrue).help("Sets the case insensitive flag for all regexes")) - .arg(Arg::new("OUTPUT").short('o').long("outputfile").action(ArgAction::Set).help("Sets the path to write the scanner results to (stdout by default)")) - .arg(Arg::new("PRETTYPRINT").long("prettyprint").action(ArgAction::SetTrue).help("Outputs the JSON in human readable format")) - .arg(Arg::new("ALLOWLIST").short('a').long("allowlist").action(ArgAction::Set).help("Sets a custom allowlist JSON file")) + .arg( + Arg::new("REGEX") + .short('r') + .long("regex") + .action(ArgAction::Set) + .value_name("REGEX") + .help("Sets a custom regex JSON file"), + ) + .arg( + Arg::new("FSPATH") + .required(true) + .action(ArgAction::Set) + .value_name("PATH") + .help("Sets the path of the directory or file to scan."), + ) + .arg( + Arg::new("NORECURSIVE") + .long("norecursive") + .action(ArgAction::SetTrue) + .help( + "Disable recursive scanning of all subdirectories underneath the supplied path", + ), + ) + .arg( + Arg::new("VERBOSE") + .short('v') + .long("verbose") + .action(ArgAction::Count) + .help("Sets the level of debugging information"), + ) + .arg( + Arg::new("ENTROPY") + .long("entropy") + .action(ArgAction::SetTrue) + .help("Enables entropy scanning"), + ) + .arg( + Arg::new("DEFAULT_ENTROPY_THRESHOLD") + .long("default_entropy_threshold") + .action(ArgAction::Set) + .default_value("0.6") + .help("Default entropy threshold (0.6 by default)"), + ) + .arg( + Arg::new("UNZIP") + .short('z') + .long("unzip") + .action(ArgAction::SetTrue) + .help("Recursively scans archives (ZIP and TAR) in memory (dangerous)"), + ) + .arg( + Arg::new("CASE") + .long("caseinsensitive") + .action(ArgAction::SetTrue) + .help("Sets the case insensitive flag for all regexes"), + ) + .arg( + Arg::new("OUTPUT") + .short('o') + .long("outputfile") + .action(ArgAction::Set) + .help("Sets the path to write the scanner results to (stdout by default)"), + ) + .arg( + Arg::new("PRETTYPRINT") + .long("prettyprint") + .action(ArgAction::SetTrue) + .help("Outputs the JSON in human readable format"), + ) + .arg( + Arg::new("ALLOWLIST") + .short('a') + .long("allowlist") + .action(ArgAction::Set) + .help("Sets a custom allowlist JSON file"), + ) .get_matches(); match run(&matches) { Ok(()) => {} diff --git a/src/bin/essex_hog.rs b/src/bin/essex_hog.rs index 86ace11..577eb54 100644 --- a/src/bin/essex_hog.rs +++ b/src/bin/essex_hog.rs @@ -33,7 +33,7 @@ extern crate clap; extern crate hyper; extern crate hyper_rustls; -use base64::{Engine as _, engine::general_purpose as Base64Engine}; +use base64::{engine::general_purpose as Base64Engine, Engine as _}; use clap::{Arg, ArgAction, ArgMatches, Command}; use encoding::all::ASCII; use encoding::types::Encoding; @@ -77,19 +77,91 @@ async fn main() { .version("1.0.11") .author("Emily Cain , Scott Cutler") .about("Confluence secret scanner in Rust.") - .arg(Arg::new("REGEX").long("regex").action(ArgAction::Set).help("Sets a custom regex JSON file")) - .arg(Arg::new("PAGEID").required(true).action(ArgAction::Set).help("The ID (e.g. 1234) of the confluence page you want to scan")) - .arg(Arg::new("URL").required(true).action(ArgAction::Set).help("Base URL of Confluence instance (e.g. https://newrelic.atlassian.net/)")) - .arg(Arg::new("VERBOSE").short('v').long("verbose").action(ArgAction::Count).help("Sets the level of debugging information")) - .arg(Arg::new("ENTROPY").long("entropy").action(ArgAction::Set).help("Enables entropy scanning")) - .arg(Arg::new("DEFAULT_ENTROPY_THRESHOLD").long("default_entropy_threshold").action(ArgAction::Set).default_value("0.6").help("Default entropy threshold (0.6 by default)")) - .arg(Arg::new("CASE").long("caseinsensitive").action(ArgAction::SetTrue).help("Sets the case insensitive flag for all regexes")) - .arg(Arg::new("OUTPUT").short('o').long("outputfile").action(ArgAction::SetTrue).help("Sets the path to write the scanner results to (stdout by default)")) - .arg(Arg::new("PRETTYPRINT").long("prettyprint").action(ArgAction::SetTrue).help("Outputs the JSON in human readable format")) - .arg(Arg::new("USERNAME").long("username").action(ArgAction::Set).conflicts_with("BEARERTOKEN").help("Confluence username (crafts basic auth header)")) - .arg(Arg::new("PASSWORD").long("password").action(ArgAction::Set).conflicts_with("BEARERTOKEN").help("Confluence password (crafts basic auth header)")) - .arg(Arg::new("BEARERTOKEN").long("authtoken").action(ArgAction::Set).conflicts_with_all(["USERNAME","PASSWORD"]).help("Confluence basic auth bearer token (instead of user & pass)")) - .arg(Arg::new("ALLOWLIST").short('a').long("allowlist").action(ArgAction::Set).help("Sets a custom allowlist JSON file")) + .arg( + Arg::new("REGEX") + .long("regex") + .action(ArgAction::Set) + .help("Sets a custom regex JSON file"), + ) + .arg( + Arg::new("PAGEID") + .required(true) + .action(ArgAction::Set) + .help("The ID (e.g. 1234) of the confluence page you want to scan"), + ) + .arg( + Arg::new("URL") + .required(true) + .action(ArgAction::Set) + .help("Base URL of Confluence instance (e.g. https://newrelic.atlassian.net/)"), + ) + .arg( + Arg::new("VERBOSE") + .short('v') + .long("verbose") + .action(ArgAction::Count) + .help("Sets the level of debugging information"), + ) + .arg( + Arg::new("ENTROPY") + .long("entropy") + .action(ArgAction::Set) + .help("Enables entropy scanning"), + ) + .arg( + Arg::new("DEFAULT_ENTROPY_THRESHOLD") + .long("default_entropy_threshold") + .action(ArgAction::Set) + .default_value("0.6") + .help("Default entropy threshold (0.6 by default)"), + ) + .arg( + Arg::new("CASE") + .long("caseinsensitive") + .action(ArgAction::SetTrue) + .help("Sets the case insensitive flag for all regexes"), + ) + .arg( + Arg::new("OUTPUT") + .short('o') + .long("outputfile") + .action(ArgAction::SetTrue) + .help("Sets the path to write the scanner results to (stdout by default)"), + ) + .arg( + Arg::new("PRETTYPRINT") + .long("prettyprint") + .action(ArgAction::SetTrue) + .help("Outputs the JSON in human readable format"), + ) + .arg( + Arg::new("USERNAME") + .long("username") + .action(ArgAction::Set) + .conflicts_with("BEARERTOKEN") + .help("Confluence username (crafts basic auth header)"), + ) + .arg( + Arg::new("PASSWORD") + .long("password") + .action(ArgAction::Set) + .conflicts_with("BEARERTOKEN") + .help("Confluence password (crafts basic auth header)"), + ) + .arg( + Arg::new("BEARERTOKEN") + .long("authtoken") + .action(ArgAction::Set) + .conflicts_with_all(["USERNAME", "PASSWORD"]) + .help("Confluence basic auth bearer token (instead of user & pass)"), + ) + .arg( + Arg::new("ALLOWLIST") + .short('a') + .long("allowlist") + .action(ArgAction::Set) + .help("Sets a custom allowlist JSON file"), + ) .get_matches(); match run(matches).await { Ok(()) => {} diff --git a/src/bin/gottingen_hog.rs b/src/bin/gottingen_hog.rs index be61d32..210a1e0 100644 --- a/src/bin/gottingen_hog.rs +++ b/src/bin/gottingen_hog.rs @@ -26,7 +26,7 @@ extern crate clap; extern crate hyper; extern crate hyper_rustls; -use base64::{Engine as _, engine::general_purpose as Base64Engine}; +use base64::{engine::general_purpose as Base64Engine, Engine as _}; use clap::{Arg, ArgAction, ArgMatches, Command}; use encoding::all::ASCII; use encoding::types::Encoding; @@ -63,19 +63,91 @@ async fn main() { .version("1.0.11") .author("Emily Cain ") .about("Jira secret scanner in Rust.") - .arg(Arg::new("REGEX").long("regex").action(ArgAction::Set).help("Sets a custom regex JSON file")) - .arg(Arg::new("JIRAID").required(true).action(ArgAction::Set).help("The ID (e.g. PROJECT-123) of the Jira issue you want to scan")) - .arg(Arg::new("VERBOSE").short('v').long("verbose").action(ArgAction::Count).help("Sets the level of debugging information")) - .arg(Arg::new("ENTROPY").long("entropy").action(ArgAction::SetTrue).help("Enables entropy scanning")) - .arg(Arg::new("DEFAULT_ENTROPY_THRESHOLD").long("default_entropy_threshold").action(ArgAction::Set).default_value("0.6").help("Default entropy threshold (0.6 by default)")) - .arg(Arg::new("CASE").long("caseinsensitive").action(ArgAction::SetTrue).help("Sets the case insensitive flag for all regexes")) - .arg(Arg::new("OUTPUT").short('o').long("outputfile").action(ArgAction::Set).help("Sets the path to write the scanner results to (stdout by default)")) - .arg(Arg::new("PRETTYPRINT").long("prettyprint").action(ArgAction::SetTrue).help("Outputs the JSON in human readable format")) - .arg(Arg::new("USERNAME").long("username").action(ArgAction::Set).conflicts_with("BEARERTOKEN").help("Jira username (crafts basic auth header)")) - .arg(Arg::new("PASSWORD").long("password").action(ArgAction::Set).conflicts_with("BEARERTOKEN").help("Jira password (crafts basic auth header)")) - .arg(Arg::new("BEARERTOKEN").long("authtoken").action(ArgAction::Set).conflicts_with_all(["USERNAME","PASSWORD"]).help("Jira basic auth bearer token (instead of user & pass)")) - .arg(Arg::new("JIRAURL").long("url").action(ArgAction::Set).help("Base URL of JIRA instance (e.g. https://jira.atlassian.net/)")) - .arg(Arg::new("ALLOWLIST").short('a').long("allowlist").action(ArgAction::Set).help("Sets a custom allowlist JSON file")) + .arg( + Arg::new("REGEX") + .long("regex") + .action(ArgAction::Set) + .help("Sets a custom regex JSON file"), + ) + .arg( + Arg::new("JIRAID") + .required(true) + .action(ArgAction::Set) + .help("The ID (e.g. PROJECT-123) of the Jira issue you want to scan"), + ) + .arg( + Arg::new("VERBOSE") + .short('v') + .long("verbose") + .action(ArgAction::Count) + .help("Sets the level of debugging information"), + ) + .arg( + Arg::new("ENTROPY") + .long("entropy") + .action(ArgAction::SetTrue) + .help("Enables entropy scanning"), + ) + .arg( + Arg::new("DEFAULT_ENTROPY_THRESHOLD") + .long("default_entropy_threshold") + .action(ArgAction::Set) + .default_value("0.6") + .help("Default entropy threshold (0.6 by default)"), + ) + .arg( + Arg::new("CASE") + .long("caseinsensitive") + .action(ArgAction::SetTrue) + .help("Sets the case insensitive flag for all regexes"), + ) + .arg( + Arg::new("OUTPUT") + .short('o') + .long("outputfile") + .action(ArgAction::Set) + .help("Sets the path to write the scanner results to (stdout by default)"), + ) + .arg( + Arg::new("PRETTYPRINT") + .long("prettyprint") + .action(ArgAction::SetTrue) + .help("Outputs the JSON in human readable format"), + ) + .arg( + Arg::new("USERNAME") + .long("username") + .action(ArgAction::Set) + .conflicts_with("BEARERTOKEN") + .help("Jira username (crafts basic auth header)"), + ) + .arg( + Arg::new("PASSWORD") + .long("password") + .action(ArgAction::Set) + .conflicts_with("BEARERTOKEN") + .help("Jira password (crafts basic auth header)"), + ) + .arg( + Arg::new("BEARERTOKEN") + .long("authtoken") + .action(ArgAction::Set) + .conflicts_with_all(["USERNAME", "PASSWORD"]) + .help("Jira basic auth bearer token (instead of user & pass)"), + ) + .arg( + Arg::new("JIRAURL") + .long("url") + .action(ArgAction::Set) + .help("Base URL of JIRA instance (e.g. https://jira.atlassian.net/)"), + ) + .arg( + Arg::new("ALLOWLIST") + .short('a') + .long("allowlist") + .action(ArgAction::Set) + .help("Sets a custom allowlist JSON file"), + ) .get_matches(); match run(matches).await { Ok(()) => {} diff --git a/src/bin/hante_hog.rs b/src/bin/hante_hog.rs index 7983089..00f495e 100644 --- a/src/bin/hante_hog.rs +++ b/src/bin/hante_hog.rs @@ -68,19 +68,91 @@ async fn main() { .version("1.0.11") .author("Joao Henrique Machado Silva ") .about("Slack secret scanner in Rust.") - .arg(Arg::new("REGEX").long("regex").action(ArgAction::Set).help("Sets a custom regex JSON file")) - .arg(Arg::new("CHANNELID").long("channelid").required(true).action(ArgAction::Set).help("The ID (e.g. C12345) of the Slack channel you want to scan")) - .arg(Arg::new("VERBOSE").short('v').long("verbose").action(ArgAction::Count).help("Sets the level of debugging information")) - .arg(Arg::new("ENTROPY").long("entropy").action(ArgAction::SetTrue).help("Enables entropy scanning")) - .arg(Arg::new("DEFAULT_ENTROPY_THRESHOLD").long("default_entropy_threshold").action(ArgAction::Set).default_value("0.6").help("Default entropy threshold (0.6 by default)")) - .arg(Arg::new("CASE").long("caseinsensitive").action(ArgAction::SetTrue).help("Sets the case insensitive flag for all regexes")) - .arg(Arg::new("OUTPUT").short('o').long("outputfile").action(ArgAction::Set).help("Sets the path to write the scanner results to (stdout by default)")) - .arg(Arg::new("PRETTYPRINT").long("prettyprint").action(ArgAction::SetTrue).help("Outputs the JSON in human readable format")) - .arg(Arg::new("BEARERTOKEN").long("authtoken").required(true).action(ArgAction::Set).help("Slack basic auth bearer token")) - .arg(Arg::new("SLACKURL").long("url").required(true).action(ArgAction::Set).help("Base URL of Slack Workspace (e.g. https://[WORKSPACE NAME].slack.com)")) - .arg(Arg::new("ALLOWLIST").short('a').long("allowlist").action(ArgAction::Set).help("Sets a custom allowlist JSON file")) - .arg(Arg::new("LATEST").long("latest").action(ArgAction::Set).help("End of time range of messages to include in search")) - .arg(Arg::new("OLDEST").long("oldest").action(ArgAction::Set).help("Start of time range of messages to include in search")) + .arg( + Arg::new("REGEX") + .long("regex") + .action(ArgAction::Set) + .help("Sets a custom regex JSON file"), + ) + .arg( + Arg::new("CHANNELID") + .long("channelid") + .required(true) + .action(ArgAction::Set) + .help("The ID (e.g. C12345) of the Slack channel you want to scan"), + ) + .arg( + Arg::new("VERBOSE") + .short('v') + .long("verbose") + .action(ArgAction::Count) + .help("Sets the level of debugging information"), + ) + .arg( + Arg::new("ENTROPY") + .long("entropy") + .action(ArgAction::SetTrue) + .help("Enables entropy scanning"), + ) + .arg( + Arg::new("DEFAULT_ENTROPY_THRESHOLD") + .long("default_entropy_threshold") + .action(ArgAction::Set) + .default_value("0.6") + .help("Default entropy threshold (0.6 by default)"), + ) + .arg( + Arg::new("CASE") + .long("caseinsensitive") + .action(ArgAction::SetTrue) + .help("Sets the case insensitive flag for all regexes"), + ) + .arg( + Arg::new("OUTPUT") + .short('o') + .long("outputfile") + .action(ArgAction::Set) + .help("Sets the path to write the scanner results to (stdout by default)"), + ) + .arg( + Arg::new("PRETTYPRINT") + .long("prettyprint") + .action(ArgAction::SetTrue) + .help("Outputs the JSON in human readable format"), + ) + .arg( + Arg::new("BEARERTOKEN") + .long("authtoken") + .required(true) + .action(ArgAction::Set) + .help("Slack basic auth bearer token"), + ) + .arg( + Arg::new("SLACKURL") + .long("url") + .required(true) + .action(ArgAction::Set) + .help("Base URL of Slack Workspace (e.g. https://[WORKSPACE NAME].slack.com)"), + ) + .arg( + Arg::new("ALLOWLIST") + .short('a') + .long("allowlist") + .action(ArgAction::Set) + .help("Sets a custom allowlist JSON file"), + ) + .arg( + Arg::new("LATEST") + .long("latest") + .action(ArgAction::Set) + .help("End of time range of messages to include in search"), + ) + .arg( + Arg::new("OLDEST") + .long("oldest") + .action(ArgAction::Set) + .help("Start of time range of messages to include in search"), + ) .get_matches(); match run(matches).await { Ok(()) => {} @@ -98,7 +170,9 @@ async fn run(arg_matches: ArgMatches) -> Result<(), SimpleError> { let secret_scanner = ssb.build(); // Reading the Slack API token from the command line - let slackauthtoken = arg_matches.get_one::("BEARERTOKEN").map(|s| s.as_str()); + let slackauthtoken = arg_matches + .get_one::("BEARERTOKEN") + .map(|s| s.as_str()); // Reading Slack Channel ID from the command line let channel_id = arg_matches .get_one::("CHANNELID") // TODO validate the format somehow @@ -114,14 +188,10 @@ async fn run(arg_matches: ArgMatches) -> Result<(), SimpleError> { let base_url = base_url_as_url.as_str(); // Reading the latest timestamp from the command line - let latest_input = arg_matches - .get_one::("LATEST") - .map(|s| s.as_str()); + let latest_input = arg_matches.get_one::("LATEST").map(|s| s.as_str()); // Reading the latest timestamp from the command line - let oldest_input = arg_matches - .get_one::("OLDEST") - .map(|s| s.as_str()); + let oldest_input = arg_matches.get_one::("OLDEST").map(|s| s.as_str()); // Still inside `async fn main`... let https = hyper_rustls::HttpsConnectorBuilder::new() @@ -136,10 +206,20 @@ async fn run(arg_matches: ArgMatches) -> Result<(), SimpleError> { // Building URL to request conversation history for the channel // TODO: Construct the URL using a URL library to avoid weird input issues? - let full_url = format!("{}/api/conversations.history?channel={}", base_url, channel_id); + let full_url = format!( + "{}/api/conversations.history?channel={}", + base_url, channel_id + ); // Retrieving the history of the channel - let json_results_array = get_channel_history_json(hyper_client, auth_string, &full_url, latest_input, oldest_input).await; + let json_results_array = get_channel_history_json( + hyper_client, + auth_string, + &full_url, + latest_input, + oldest_input, + ) + .await; // WARNING: This method requires storing ALL the slack channel history JSON in memory at once // TODO: Re-write these methods to scan each JSON API request - to conserve memory usage @@ -148,11 +228,7 @@ async fn run(arg_matches: ArgMatches) -> Result<(), SimpleError> { for json_results in json_results_array.iter() { // Parsing the messages as an array - let messages = json_results - .get("messages") - .unwrap() - .as_array() - .unwrap(); + let messages = json_results.get("messages").unwrap().as_array().unwrap(); // find secrets in each message for message in messages { @@ -161,12 +237,21 @@ async fn run(arg_matches: ArgMatches) -> Result<(), SimpleError> { let location = format!( "message type {} by {} on {}", message.get("type").unwrap(), - message.get("user").unwrap_or(&Value::String("".to_string())), + message + .get("user") + .unwrap_or(&Value::String("".to_string())), message.get("ts").unwrap() ); let message_text = message.get("text").unwrap().as_str().unwrap().as_bytes(); - let message_findings = get_findings(&secret_scanner, base_url, channel_id, ts, message_text, location); + let message_findings = get_findings( + &secret_scanner, + base_url, + channel_id, + ts, + message_text, + location, + ); secrets.extend(message_findings); } } @@ -183,7 +268,6 @@ async fn run(arg_matches: ArgMatches) -> Result<(), SimpleError> { } } - // TODO: move this to a separate file /// get_channel_history_json uses a hyper::client object to perform a POST on the full_url and return parsed serde JSON data async fn get_channel_history_json<'a, C>( @@ -237,7 +321,9 @@ where if status != StatusCode::OK { panic!( "Request to {} failed with code {:?}: {}", - full_url_mod.clone(), status, response_body + full_url_mod.clone(), + status, + response_body ) } @@ -247,16 +333,25 @@ where if !ok { panic!( "Request to {} failed with error {:?}: {}", - full_url_mod.clone(), json_results["error"], response_body + full_url_mod.clone(), + json_results["error"], + response_body ) } has_more = json_results.get("has_more").unwrap().as_bool().unwrap(); - if has_more { // TODO: Cleanup weird borrowing issues? - let rm = json_results.get("response_metadata").unwrap().as_object().unwrap().clone(); - cursor = Some(String::from(rm.get("next_cursor").unwrap().as_str().unwrap())); + if has_more { + // TODO: Cleanup weird borrowing issues? + let rm = json_results + .get("response_metadata") + .unwrap() + .as_object() + .unwrap() + .clone(); + cursor = Some(String::from( + rm.get("next_cursor").unwrap().as_str().unwrap(), + )); } output.push(json_results); - } output } @@ -271,12 +366,11 @@ fn get_findings( description: &[u8], location: String, ) -> Vec { - let lines = description.split(|&x| (x as char) == '\n'); let mut secrets: Vec = Vec::new(); // Building web links for Slack messages - // https://.slack.com/archives/ + // https://.slack.com/archives/ let msg_id = str::replace(ts, ".", ""); let web_link = format!("{}/archives/{}/p{}", base_url, channel_id, msg_id); @@ -284,7 +378,8 @@ fn get_findings( for new_line in lines { debug!("{:?}", std::str::from_utf8(new_line)); // Builds a BTreeMap of the findings - let matches_map: BTreeMap> = secret_scanner.matches_entropy(new_line); + let matches_map: BTreeMap> = + secret_scanner.matches_entropy(new_line); // Iterate over the findings and add them to the list of findings to return for (reason, match_iterator) in matches_map { diff --git a/src/git_scanning.rs b/src/git_scanning.rs index 5e428ae..848a79a 100644 --- a/src/git_scanning.rs +++ b/src/git_scanning.rs @@ -49,12 +49,12 @@ use encoding::{DecoderTrap, Encoding}; use git2::{Commit, DiffFormat, Tree}; use git2::{DiffOptions, Repository, Time}; use log::{self, debug, info}; +use rusty_hog_scanner::{RustyHogMatch, SecretScanner}; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, HashSet}; use std::hash::{Hash, Hasher}; use std::path::Path; use std::{fmt, str}; -use rusty_hog_scanner::{RustyHogMatch, SecretScanner}; use url::{ParseError, Url}; #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Hash, Clone, Default)] From f17e6b5884f90aa1505f12422caf6d4b3ce1bebd Mon Sep 17 00:00:00 2001 From: Zach Carlson Date: Fri, 22 Dec 2023 13:22:58 -0600 Subject: [PATCH 18/26] rustfmt rusty-hog-scanner crate --- crates/rusty-hog-scanner/src/lib.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/crates/rusty-hog-scanner/src/lib.rs b/crates/rusty-hog-scanner/src/lib.rs index 45989fb..450e9dd 100644 --- a/crates/rusty-hog-scanner/src/lib.rs +++ b/crates/rusty-hog-scanner/src/lib.rs @@ -68,7 +68,7 @@ extern crate clap; use anyhow::Result; -use base64::{Engine as _, engine::general_purpose as Base64Engine}; +use base64::{engine::general_purpose as Base64Engine, Engine as _}; use clap::ArgMatches; use log::{self, debug, error, info, LevelFilter}; use regex::bytes::{Match, Matches, Regex, RegexBuilder}; @@ -1048,12 +1048,12 @@ impl PartialEq for SecretScanner { && self.regex_map.keys().eq(other.regex_map.keys()) && self.pretty_print == other.pretty_print && match self.output_path.as_ref() { - None => other.output_path.is_none(), - Some(s) => match other.output_path.as_ref() { - None => false, - Some(t) => *s == *t, - }, - } + None => other.output_path.is_none(), + Some(s) => match other.output_path.as_ref() { + None => false, + Some(t) => *s == *t, + }, + } } } @@ -1109,7 +1109,7 @@ mod tests { not_so_secret_but_has_the_word_secret_and_is_long "#, ) - .into_bytes(); + .into_bytes(); let output = SecretScanner::entropy_findings(test_string.as_slice(), 0.6); // println!("{:?}", output); assert_eq!(output.len(), 1); @@ -1164,7 +1164,7 @@ mod tests { not_so_secret_but_has_the_word_secret_and_is_long "#, ) - .into_bytes(); + .into_bytes(); let mut findings: Vec<(String, String)> = Vec::new(); // Main loop - split the data based on newlines, then run get_matches() on each line, // then make a list of findings in output @@ -1209,7 +1209,7 @@ mod tests { @ "#, ) - .into_bytes(); + .into_bytes(); let mut findings: Vec<(String, String)> = Vec::new(); // Main loop - split the data based on newlines, then run get_matches() on each line, // then make a list of findings in output From ad455ad1f4d843f86573c1c4651bd4ef5b4eefb1 Mon Sep 17 00:00:00 2001 From: Zach Carlson Date: Fri, 22 Dec 2023 13:28:39 -0600 Subject: [PATCH 19/26] x86_64-linux-musl: install openssl dev before compiling --- Cargo.toml | 3 +++ Dockerfile.lambda | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 Dockerfile.lambda diff --git a/Cargo.toml b/Cargo.toml index 19006c7..43eb08a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,3 +53,6 @@ escargot = "0.5.0" [profile.release] lto = true codegen-units = 1 + +[package.metadata.cross.target.x86_64-unknown-linux-musl] +dockerfile = "Dockerfile.lambda" diff --git a/Dockerfile.lambda b/Dockerfile.lambda new file mode 100644 index 0000000..171e91d --- /dev/null +++ b/Dockerfile.lambda @@ -0,0 +1,16 @@ +ARG CROSS_BASE_IMAGE +FROM ${CROSS_BASE_IMAGE} +# Note that we're assuming an Ubuntu-based image in all cases though + +ARG CFLAGS="" +ARG LDFLAGS="" + +ARG OPENSSL_BUILD_VER=3.0.12 +RUN cd /usr/local/src/ && curl -sLO https://www.openssl.org/source/openssl-${OPENSSL_BUILD_VER}.tar.gz && \ + tar xzvf openssl-${OPENSSL_BUILD_VER}.tar.gz && cd openssl-${OPENSSL_BUILD_VER} && \ + CROSS_COMPILE="x86_64-linux-musl-" ./Configure --prefix=/usr/local/openssl-${OPENSSL_BUILD_VER} linux-x86_64 && make && make install + +ENV OPENSSL_DIR="/usr/local/openssl-${OPENSSL_BUILD_VER}" +ENV OPENSSL_STATIC="/usr/local/openssl-${OPENSSL_BUILD_VER}/lib" +ENV CFLAGS="${CFLAGS} -I/usr/local/openssl-${OPENSSL_BUILD_VER}/include" +ENV LDFLAGS="${LDFLAGS} -L/usr/local/openssl-${OPENSSL_BUILD_VER}/lib64" From 913ed26505be94f29007d89c447d4425c9ffed73 Mon Sep 17 00:00:00 2001 From: Zach Carlson Date: Fri, 22 Dec 2023 14:47:16 -0600 Subject: [PATCH 20/26] arg DEFAULT_ENTROPY_THRESHOLD fix --- src/bin/duroc_hog.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/bin/duroc_hog.rs b/src/bin/duroc_hog.rs index 352bca6..1889835 100644 --- a/src/bin/duroc_hog.rs +++ b/src/bin/duroc_hog.rs @@ -111,7 +111,6 @@ fn main() { Arg::new("DEFAULT_ENTROPY_THRESHOLD") .long("default_entropy_threshold") .action(ArgAction::Set) - .default_value("0.6") .help("Default entropy threshold (0.6 by default)"), ) .arg( @@ -408,7 +407,7 @@ mod tests { use std::io::Result; use std::io::Write; use std::process::Output; - use tempfile::{NamedTempFile, TempDir}; + use tempfile::{NamedTempFile, TempDir, tempdir}; fn run_command_in_dir(dir: &TempDir, command: &str, args: &[&str]) -> Result { let dir_path = dir.path().to_str().unwrap(); @@ -433,7 +432,7 @@ mod tests { #[test] fn does_not_scan_output_file() { - let temp_dir = TempDir::new().unwrap(); + let temp_dir = tempdir().expect("couldn't make tempdir"); write_temp_file( &temp_dir, @@ -444,12 +443,12 @@ mod tests { let cmd_args = ["-o", "output_file.txt", "."]; run_command_in_dir(&temp_dir, "duroc_hog", &cmd_args).unwrap(); - run_command_in_dir(&temp_dir, "duroc_hog", &cmd_args).unwrap(); let text = read_temp_file(&temp_dir, "output_file.txt"); println!("{}", text); + temp_dir.close().expect("couldn't close tempdir"); assert!(text.contains("\"path\":\"./insecure-file.txt\"")); assert!(!text.contains("output_file.txt")); @@ -457,7 +456,7 @@ mod tests { #[test] fn allowlist_json_file_prevents_output() { - let temp_dir = TempDir::new().unwrap(); + let temp_dir = tempdir().expect("couldn't make tempdir"); let mut allowlist_temp_file = NamedTempFile::new().unwrap(); let json = r#" { @@ -480,7 +479,8 @@ mod tests { ]; let output = run_command_in_dir(&temp_dir, "duroc_hog", &cmd_args).unwrap(); - - assert_eq!("[]\n", str::from_utf8(&output.stdout).unwrap()); + temp_dir.close().expect("couldn't close tempdir"); + let prg_out = str::from_utf8(&output.stdout).unwrap(); + assert_eq!("[]\n", prg_out); } } From eb75fdccf77b53c4f18303ddb36cfd10aed9cc2f Mon Sep 17 00:00:00 2001 From: Zach Carlson Date: Fri, 22 Dec 2023 14:57:43 -0600 Subject: [PATCH 21/26] default_entropy_threshold parse update --- src/bin/ankamali_hog.rs | 2 ++ src/bin/berkshire_hog.rs | 2 +- src/bin/choctaw_hog.rs | 2 +- src/bin/duroc_hog.rs | 2 ++ src/bin/essex_hog.rs | 1 + src/bin/gottingen_hog.rs | 1 + src/bin/hante_hog.rs | 1 + 7 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/bin/ankamali_hog.rs b/src/bin/ankamali_hog.rs index 8e0fd6f..2e2ec2b 100644 --- a/src/bin/ankamali_hog.rs +++ b/src/bin/ankamali_hog.rs @@ -74,6 +74,8 @@ async fn main() { Arg::new("DEFAULT_ENTROPY_THRESHOLD") .long("default_entropy_threshold") .action(ArgAction::Set) + .default_value("0.6") + .value_parser(clap::value_parser!(f32)) .help("Default entropy threshold (0.6 by default)"), ) .arg( diff --git a/src/bin/berkshire_hog.rs b/src/bin/berkshire_hog.rs index d716e2a..40bf138 100644 --- a/src/bin/berkshire_hog.rs +++ b/src/bin/berkshire_hog.rs @@ -53,7 +53,7 @@ fn main() { .arg(Arg::new("RECURSIVE").short('r').long("recursive").action(ArgAction::SetTrue).help("Recursively scans files under the prefix")) .arg(Arg::new("VERBOSE").short('v').long("verbose").action(ArgAction::Count).help("Sets the level of debugging information")) .arg(Arg::new("ENTROPY").long("entropy").action(ArgAction::SetTrue).help("Enables entropy scanning")) - .arg(Arg::new("DEFAULT_ENTROPY_THRESHOLD").long("default_entropy_threshold").action(ArgAction::Set).default_value("0.6").help("Default entropy threshold (0.6 by default)")) + .arg(Arg::new("DEFAULT_ENTROPY_THRESHOLD").long("default_entropy_threshold").action(ArgAction::Set).default_value("0.6").value_parser(clap::value_parser!(f32)).help("Default entropy threshold (0.6 by default)")) .arg(Arg::new("CASE").long("caseinsensitive").action(ArgAction::SetTrue).help("Sets the case insensitive flag for all regexes")) .arg(Arg::new("OUTPUT").short('o').long("outputfile").action(ArgAction::Set).help("Sets the path to write the scanner results to (stdout by default)")) .arg(Arg::new("PRETTYPRINT").long("prettyprint").action(ArgAction::SetTrue).help("Outputs the JSON in human readable format")) diff --git a/src/bin/choctaw_hog.rs b/src/bin/choctaw_hog.rs index a9ece73..54b7d32 100644 --- a/src/bin/choctaw_hog.rs +++ b/src/bin/choctaw_hog.rs @@ -58,7 +58,7 @@ fn main() { .arg(Arg::new("GITPATH").required(true).action(ArgAction::Set).value_name("GIT_PATH").help("Sets the path (or URL) of the Git repo to scan. SSH links must include username (git@)")) .arg(Arg::new("VERBOSE").short('v').long("verbose").action(ArgAction::Count).help("Sets the level of debugging information")) .arg(Arg::new("ENTROPY").long("entropy").action(ArgAction::SetTrue).help("Enables entropy scanning")) - .arg(Arg::new("DEFAULT_ENTROPY_THRESHOLD").long("default_entropy_threshold").action(ArgAction::Set).default_value("0.6").help("Default entropy threshold (0.6 by default)")) + .arg(Arg::new("DEFAULT_ENTROPY_THRESHOLD").long("default_entropy_threshold").action(ArgAction::Set).default_value("0.6").value_parser(clap::value_parser!(f32)).help("Default entropy threshold (0.6 by default)")) .arg(Arg::new("CASE").long("caseinsensitive").action(ArgAction::SetTrue).help("Sets the case insensitive flag for all regexes")) .arg(Arg::new("OUTPUT").short('o').long("outputfile").action(ArgAction::Set).help("Sets the path to write the scanner results to (stdout by default)")) .arg(Arg::new("PRETTYPRINT").long("prettyprint").action(ArgAction::SetTrue).help("Outputs the JSON in human readable format")) diff --git a/src/bin/duroc_hog.rs b/src/bin/duroc_hog.rs index 1889835..186704d 100644 --- a/src/bin/duroc_hog.rs +++ b/src/bin/duroc_hog.rs @@ -111,6 +111,8 @@ fn main() { Arg::new("DEFAULT_ENTROPY_THRESHOLD") .long("default_entropy_threshold") .action(ArgAction::Set) + .default_value("0.6") + .value_parser(clap::value_parser!(f32)) .help("Default entropy threshold (0.6 by default)"), ) .arg( diff --git a/src/bin/essex_hog.rs b/src/bin/essex_hog.rs index 577eb54..37a3da4 100644 --- a/src/bin/essex_hog.rs +++ b/src/bin/essex_hog.rs @@ -113,6 +113,7 @@ async fn main() { .long("default_entropy_threshold") .action(ArgAction::Set) .default_value("0.6") + .value_parser(clap::value_parser!(f32)) .help("Default entropy threshold (0.6 by default)"), ) .arg( diff --git a/src/bin/gottingen_hog.rs b/src/bin/gottingen_hog.rs index 210a1e0..f03589f 100644 --- a/src/bin/gottingen_hog.rs +++ b/src/bin/gottingen_hog.rs @@ -93,6 +93,7 @@ async fn main() { .long("default_entropy_threshold") .action(ArgAction::Set) .default_value("0.6") + .value_parser(clap::value_parser!(f32)) .help("Default entropy threshold (0.6 by default)"), ) .arg( diff --git a/src/bin/hante_hog.rs b/src/bin/hante_hog.rs index 00f495e..122a8a4 100644 --- a/src/bin/hante_hog.rs +++ b/src/bin/hante_hog.rs @@ -99,6 +99,7 @@ async fn main() { .long("default_entropy_threshold") .action(ArgAction::Set) .default_value("0.6") + .value_parser(clap::value_parser!(f32)) .help("Default entropy threshold (0.6 by default)"), ) .arg( From 76b1c1534d60fbf5b9b0a9b7bd0e16666f6d1dba Mon Sep 17 00:00:00 2001 From: Zach Carlson Date: Fri, 22 Dec 2023 15:46:57 -0600 Subject: [PATCH 22/26] Fix tests, cargo fmt again --- src/aws_scanning.rs | 2 +- src/bin/duroc_hog.rs | 2 +- src/google_scanning.rs | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/aws_scanning.rs b/src/aws_scanning.rs index c1f214e..25eb009 100644 --- a/src/aws_scanning.rs +++ b/src/aws_scanning.rs @@ -44,7 +44,7 @@ //! let region: Region = Region::UsWest2; //! let bucket: Bucket = match Bucket::new(bucket_string, region, credentials) { //! Ok(r) => r, -//! Err(e) => panic!(e) +//! Err(e) => panic!("{}", e) //! }; //! let results = s3s.scan_s3_file(bucket, "s3://testbucket1/727463.json").unwrap(); //! assert_eq!(results.len(), 0); diff --git a/src/bin/duroc_hog.rs b/src/bin/duroc_hog.rs index 186704d..88281fe 100644 --- a/src/bin/duroc_hog.rs +++ b/src/bin/duroc_hog.rs @@ -409,7 +409,7 @@ mod tests { use std::io::Result; use std::io::Write; use std::process::Output; - use tempfile::{NamedTempFile, TempDir, tempdir}; + use tempfile::{tempdir, NamedTempFile, TempDir}; fn run_command_in_dir(dir: &TempDir, command: &str, args: &[&str]) -> Result { let dir_path = dir.path().to_str().unwrap(); diff --git a/src/google_scanning.rs b/src/google_scanning.rs index 578d3e1..8876475 100644 --- a/src/google_scanning.rs +++ b/src/google_scanning.rs @@ -55,7 +55,7 @@ //! # .build() //! # .await //! # .expect("failed to create authenticator (try deleting temp_token and restarting)"); -//! let hub = DriveHub::new(hyper::Client::builder().build(hyper_rustls::HttpsConnector::with_native_roots()), auth); +//! let hub = DriveHub::new(hyper::Client::builder().build(hyper_rustls::HttpsConnectorBuilder::new().with_native_roots().https_only().enable_all_versions().build()), auth); //! //! // get some initial info about the file //! let gdriveinfo = GDriveFileInfo::new("1FCdv-FQAgfNenGbvXfiplT7S5OFj0oqrFQ1_KwD_n90", &hub).await.unwrap(); @@ -94,7 +94,7 @@ use tokio::io::{AsyncRead, AsyncWrite}; /// ``` /// # use rusty_hogs::google_scanning::GDriveFinding; /// let gdf: GDriveFinding = GDriveFinding { -/// date: String::from("2019-12-21T16:32:31+00:00"), +/// date: chrono::DateTime::parse_from_rfc3339("2019-12-21T16:32:31+00:00").unwrap().into(), /// diff: String::from("context around finding"), /// path: String::from("GDrive folder path"), /// strings_found: Vec::new(), @@ -133,7 +133,7 @@ pub struct GDriveScanner { /// let gdfi: GDriveFileInfo = GDriveFileInfo { /// file_id: String::from("GDrive file ID"), /// mime_type: String::from("MIME"), -/// modified_time: String::from("context around finding"), +/// modified_time: chrono::Utc::now(), /// web_link: String::from("context around finding"), /// parents: Vec::new(), /// name: String::from("context around finding"), From affa345f223154eab95e97c1f0d378c969bc29d0 Mon Sep 17 00:00:00 2001 From: Zach Carlson Date: Fri, 19 Jan 2024 15:16:50 -0600 Subject: [PATCH 23/26] the git-scanning test needs git history to work --- .github/workflows/testing.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index de7d794..e9655d4 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -56,6 +56,9 @@ jobs: steps: - name: Checkout sources uses: actions/checkout@v3 + with: + # git-scanning tests need entire git history to work + fetch-depth: 0 - name: Install ${{ matrix.rust }}-${{ matrix.target }} toolchain uses: actions-rs/toolchain@v1 with: From b3b752077bcd3556a7e353d97807e4a730c86e0b Mon Sep 17 00:00:00 2001 From: Zach Carlson Date: Fri, 19 Jan 2024 15:37:41 -0600 Subject: [PATCH 24/26] insecure-file path affected by Windows --- src/bin/duroc_hog.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bin/duroc_hog.rs b/src/bin/duroc_hog.rs index 88281fe..88277ea 100644 --- a/src/bin/duroc_hog.rs +++ b/src/bin/duroc_hog.rs @@ -452,7 +452,7 @@ mod tests { println!("{}", text); temp_dir.close().expect("couldn't close tempdir"); - assert!(text.contains("\"path\":\"./insecure-file.txt\"")); + assert!(text.contains("\"path\":\"./insecure-file.txt\"") || text.contains("\"path\":\".\\insecure-file.txt\"")); assert!(!text.contains("output_file.txt")); } From 17795de4425c865354c7fbc253c2be7575c792fa Mon Sep 17 00:00:00 2001 From: Zach Carlson Date: Fri, 19 Jan 2024 15:41:11 -0600 Subject: [PATCH 25/26] cargo fmt --- src/bin/duroc_hog.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/bin/duroc_hog.rs b/src/bin/duroc_hog.rs index 88277ea..0da44c6 100644 --- a/src/bin/duroc_hog.rs +++ b/src/bin/duroc_hog.rs @@ -452,7 +452,10 @@ mod tests { println!("{}", text); temp_dir.close().expect("couldn't close tempdir"); - assert!(text.contains("\"path\":\"./insecure-file.txt\"") || text.contains("\"path\":\".\\insecure-file.txt\"")); + assert!( + text.contains("\"path\":\"./insecure-file.txt\"") + || text.contains("\"path\":\".\\insecure-file.txt\"") + ); assert!(!text.contains("output_file.txt")); } From 8272a01b3923caaa841db1d878e6f75da9401568 Mon Sep 17 00:00:00 2001 From: Zach Carlson Date: Fri, 19 Jan 2024 15:51:57 -0600 Subject: [PATCH 26/26] two double-backslashes one for JSON, one for Windows --- src/bin/duroc_hog.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bin/duroc_hog.rs b/src/bin/duroc_hog.rs index 0da44c6..7642208 100644 --- a/src/bin/duroc_hog.rs +++ b/src/bin/duroc_hog.rs @@ -454,7 +454,7 @@ mod tests { assert!( text.contains("\"path\":\"./insecure-file.txt\"") - || text.contains("\"path\":\".\\insecure-file.txt\"") + || text.contains("\"path\":\".\\\\insecure-file.txt\"") ); assert!(!text.contains("output_file.txt")); }