From 4327a5c8d2dca8621dbc6f1675984659e3717615 Mon Sep 17 00:00:00 2001 From: faith Date: Mon, 30 Dec 2024 09:39:21 +0800 Subject: [PATCH 1/3] refactor: move shuf/rand_read_adapter to uucore --- Cargo.lock | 2 + src/uu/shuf/Cargo.toml | 2 +- src/uu/shuf/src/shuf.rs | 42 ++----------------- src/uucore/Cargo.toml | 3 ++ src/uucore/src/lib/features.rs | 2 + .../src/lib/features/rand_read.rs} | 38 ++++++++++++++++- src/uucore/src/lib/lib.rs | 2 + 7 files changed, 50 insertions(+), 41 deletions(-) rename src/{uu/shuf/src/rand_read_adapter.rs => uucore/src/lib/features/rand_read.rs} (80%) diff --git a/Cargo.lock b/Cargo.lock index 957a797e40b..8c904410dc2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3477,6 +3477,8 @@ dependencies = [ "number_prefix", "once_cell", "os_display", + "rand", + "rand_core", "regex", "sha1", "sha2", diff --git a/src/uu/shuf/Cargo.toml b/src/uu/shuf/Cargo.toml index f8b887b7e87..0a0bd9d78f0 100644 --- a/src/uu/shuf/Cargo.toml +++ b/src/uu/shuf/Cargo.toml @@ -21,7 +21,7 @@ clap = { workspace = true } memchr = { workspace = true } rand = { workspace = true } rand_core = { workspace = true } -uucore = { workspace = true } +uucore = { workspace = true, features = ["rand-read"] } [[bin]] name = "shuf" diff --git a/src/uu/shuf/src/shuf.rs b/src/uu/shuf/src/shuf.rs index 2d8023448a0..8f39d777b38 100644 --- a/src/uu/shuf/src/shuf.rs +++ b/src/uu/shuf/src/shuf.rs @@ -8,17 +8,16 @@ use clap::{crate_version, Arg, ArgAction, Command}; use memchr::memchr_iter; use rand::prelude::SliceRandom; -use rand::{Rng, RngCore}; +use rand::Rng; use std::collections::HashSet; use std::fs::File; use std::io::{stdin, stdout, BufReader, BufWriter, Error, Read, Write}; use std::ops::RangeInclusive; use uucore::display::Quotable; use uucore::error::{FromIo, UResult, USimpleError, UUsageError}; +use uucore::rand_read::{ReadRng, WrappedRng}; use uucore::{format_usage, help_about, help_usage}; -mod rand_read_adapter; - enum Mode { Default(String), Echo(Vec), @@ -433,7 +432,7 @@ fn shuf_exec(input: &mut impl Shufable, opts: Options) -> UResult<()> { Some(r) => { let file = File::open(&r[..]) .map_err_context(|| format!("failed to open random source {}", r.quote()))?; - WrappedRng::RngFile(rand_read_adapter::ReadRng::new(file)) + WrappedRng::RngFile(ReadRng::new(file)) } None => WrappedRng::RngDefault(rand::thread_rng()), }; @@ -494,41 +493,6 @@ fn parse_head_count(headcounts: Vec) -> Result { Ok(result) } -enum WrappedRng { - RngFile(rand_read_adapter::ReadRng), - RngDefault(rand::rngs::ThreadRng), -} - -impl RngCore for WrappedRng { - fn next_u32(&mut self) -> u32 { - match self { - Self::RngFile(r) => r.next_u32(), - Self::RngDefault(r) => r.next_u32(), - } - } - - fn next_u64(&mut self) -> u64 { - match self { - Self::RngFile(r) => r.next_u64(), - Self::RngDefault(r) => r.next_u64(), - } - } - - fn fill_bytes(&mut self, dest: &mut [u8]) { - match self { - Self::RngFile(r) => r.fill_bytes(dest), - Self::RngDefault(r) => r.fill_bytes(dest), - } - } - - fn try_fill_bytes(&mut self, dest: &mut [u8]) -> Result<(), rand::Error> { - match self { - Self::RngFile(r) => r.try_fill_bytes(dest), - Self::RngDefault(r) => r.try_fill_bytes(dest), - } - } -} - #[cfg(test)] // Since the computed value is a bool, it is more readable to write the expected value out: #[allow(clippy::bool_assert_comparison)] diff --git a/src/uucore/Cargo.toml b/src/uucore/Cargo.toml index 92eaf08536c..c13c9b3c9d3 100644 --- a/src/uucore/Cargo.toml +++ b/src/uucore/Cargo.toml @@ -26,6 +26,8 @@ dunce = { version = "1.0.4", optional = true } wild = "2.2.1" glob = { workspace = true } lazy_static = "1.4.0" +rand = { workspace = true } +rand_core = { workspace = true } # * optional itertools = { workspace = true, optional = true } thiserror = { workspace = true, optional = true } @@ -92,6 +94,7 @@ pipes = [] process = ["libc"] proc-info = ["tty", "walkdir"] quoting-style = [] +rand-read = [] ranges = [] ringbuffer = [] signals = [] diff --git a/src/uucore/src/lib/features.rs b/src/uucore/src/lib/features.rs index ef5be724d9f..0d60de16b5b 100644 --- a/src/uucore/src/lib/features.rs +++ b/src/uucore/src/lib/features.rs @@ -24,6 +24,8 @@ pub mod fsext; pub mod lines; #[cfg(feature = "quoting-style")] pub mod quoting_style; +#[cfg(feature = "rand-read")] +pub mod rand_read; #[cfg(feature = "ranges")] pub mod ranges; #[cfg(feature = "ringbuffer")] diff --git a/src/uu/shuf/src/rand_read_adapter.rs b/src/uucore/src/lib/features/rand_read.rs similarity index 80% rename from src/uu/shuf/src/rand_read_adapter.rs rename to src/uucore/src/lib/features/rand_read.rs index 728bc0cfbbd..d396d39b4e9 100644 --- a/src/uu/shuf/src/rand_read_adapter.rs +++ b/src/uucore/src/lib/features/rand_read.rs @@ -13,8 +13,8 @@ //! A wrapper around any Read to treat it as an RNG. -use std::fmt; use std::io::Read; +use std::{fmt, fs::File}; use rand_core::{impls, Error, RngCore}; @@ -89,6 +89,42 @@ impl std::error::Error for ReadError { } } +/// Generic interface for both ThreadRng and ReadRng +pub enum WrappedRng { + RngFile(ReadRng), + RngDefault(rand::rngs::ThreadRng), +} + +impl RngCore for WrappedRng { + fn next_u32(&mut self) -> u32 { + match self { + Self::RngFile(r) => r.next_u32(), + Self::RngDefault(r) => r.next_u32(), + } + } + + fn next_u64(&mut self) -> u64 { + match self { + Self::RngFile(r) => r.next_u64(), + Self::RngDefault(r) => r.next_u64(), + } + } + + fn fill_bytes(&mut self, dest: &mut [u8]) { + match self { + Self::RngFile(r) => r.fill_bytes(dest), + Self::RngDefault(r) => r.fill_bytes(dest), + } + } + + fn try_fill_bytes(&mut self, dest: &mut [u8]) -> Result<(), rand::Error> { + match self { + Self::RngFile(r) => r.try_fill_bytes(dest), + Self::RngDefault(r) => r.try_fill_bytes(dest), + } + } +} + #[cfg(test)] mod test { use std::println; diff --git a/src/uucore/src/lib/lib.rs b/src/uucore/src/lib/lib.rs index 9516b5e1bf6..b6d8c9baf43 100644 --- a/src/uucore/src/lib/lib.rs +++ b/src/uucore/src/lib/lib.rs @@ -56,6 +56,8 @@ pub use crate::features::fs; pub use crate::features::lines; #[cfg(feature = "quoting-style")] pub use crate::features::quoting_style; +#[cfg(feature = "rand-read")] +pub use crate::features::rand_read; #[cfg(feature = "ranges")] pub use crate::features::ranges; #[cfg(feature = "ringbuffer")] From 4fe7dc91cddd3f5277fac2b139ac233e84343109 Mon Sep 17 00:00:00 2001 From: faith Date: Mon, 30 Dec 2024 09:39:48 +0800 Subject: [PATCH 2/3] feat: implement --random-source for shred --- src/uu/shred/Cargo.toml | 2 +- src/uu/shred/src/shred.rs | 27 ++++++++++++++++++++++++++- tests/by-util/test_shred.rs | 24 ++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 2 deletions(-) diff --git a/src/uu/shred/Cargo.toml b/src/uu/shred/Cargo.toml index 10394565a37..66ea5251695 100644 --- a/src/uu/shred/Cargo.toml +++ b/src/uu/shred/Cargo.toml @@ -19,7 +19,7 @@ path = "src/shred.rs" [dependencies] clap = { workspace = true } rand = { workspace = true } -uucore = { workspace = true } +uucore = { workspace = true, features = ["rand-read"] } libc = { workspace = true } [[bin]] diff --git a/src/uu/shred/src/shred.rs b/src/uu/shred/src/shred.rs index 763d6cfd4bf..f66a9dbc7fd 100644 --- a/src/uu/shred/src/shred.rs +++ b/src/uu/shred/src/shred.rs @@ -17,6 +17,7 @@ use std::path::{Path, PathBuf}; use uucore::display::Quotable; use uucore::error::{FromIo, UResult, USimpleError, UUsageError}; use uucore::parse_size::parse_size_u64; +use uucore::rand_read::{ReadRng, WrappedRng}; use uucore::shortcut_value_parser::ShortcutValueParser; use uucore::{format_usage, help_about, help_section, help_usage, show_error, show_if_err}; @@ -34,6 +35,7 @@ pub mod options { pub const VERBOSE: &str = "verbose"; pub const EXACT: &str = "exact"; pub const ZERO: &str = "zero"; + pub const RANDOM_SOURCE: &str = "random-source"; pub mod remove { pub const UNLINK: &str = "unlink"; @@ -261,6 +263,9 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { let exact = matches.get_flag(options::EXACT) || size.is_some(); let zero = matches.get_flag(options::ZERO); let verbose = matches.get_flag(options::VERBOSE); + let random_source = matches + .get_one::(options::RANDOM_SOURCE) + .map(String::from); for path_str in matches.get_many::(options::FILE).unwrap() { show_if_err!(wipe_file( @@ -272,6 +277,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { zero, verbose, force, + &random_source )); } Ok(()) @@ -357,6 +363,13 @@ pub fn uu_app() -> Command { .action(ArgAction::Append) .value_hint(clap::ValueHint::FilePath), ) + .arg( + Arg::new(options::RANDOM_SOURCE) + .long(options::RANDOM_SOURCE) + .value_name("FILE") + .help("get random bytes from FILE") + .value_hint(clap::ValueHint::FilePath), + ) } fn get_size(size_str_opt: Option) -> Option { @@ -392,6 +405,7 @@ fn wipe_file( zero: bool, verbose: bool, force: bool, + random_source: &Option, ) -> UResult<()> { // Get these potential errors out of the way first let path = Path::new(path_str); @@ -452,7 +466,17 @@ fn wipe_file( for pattern in PATTERNS.into_iter().take(remainder) { pass_sequence.push(PassType::Pattern(pattern)); } - let mut rng = rand::thread_rng(); + + let mut rng = match random_source { + Some(r) => { + let file = File::open(&r[..]).map_err_context(|| { + format!("failed to open random source {}", r.quote()) + })?; + WrappedRng::RngFile(ReadRng::new(file)) + } + None => WrappedRng::RngDefault(rand::thread_rng()), + }; + pass_sequence.shuffle(&mut rng); // randomize the order of application let n_random = 3 + n_passes / 10; // Minimum 3 random passes; ratio of 10 after @@ -501,6 +525,7 @@ fn wipe_file( do_remove(path, path_str, verbose, remove_method) .map_err_context(|| format!("{}: failed to remove file", path.maybe_quote()))?; } + Ok(()) } diff --git a/tests/by-util/test_shred.rs b/tests/by-util/test_shred.rs index 82e421839ae..6da1f1e3304 100644 --- a/tests/by-util/test_shred.rs +++ b/tests/by-util/test_shred.rs @@ -181,6 +181,30 @@ fn test_shred_empty() { assert!(!at.file_exists(file_a)); } +#[test] +fn test_random_source() { + let (at, mut ucmd) = at_and_ucmd!(); + + let file = "test_random_source"; + let file_original_content = "test_shred file content"; + + at.write(file, file_original_content); + + // Write to random file + let random_file = "test_random_file"; + at.touch(random_file); + at.write(random_file, "random contents"); + + ucmd.arg("--random-source=test_random_file") + .arg(file) + .succeeds(); + + // File exists + assert!(at.file_exists(file)); + // File is obfuscated + assert!(at.read_bytes(file) != file_original_content.as_bytes()); +} + #[test] #[cfg(all(unix, feature = "chmod"))] fn test_shred_fail_no_perm() { From fb915a4fd8d330486100dc164eeec81756831654 Mon Sep 17 00:00:00 2001 From: faith Date: Sun, 5 Jan 2025 21:33:51 +0800 Subject: [PATCH 3/3] wip: add more checks --- src/uu/shred/src/shred.rs | 61 ++++++++++++++++++++++++---- src/uu/stdbuf/src/libstdbuf/build.rs | 5 ++- tests/by-util/test_shred.rs | 21 +++++++++- 3 files changed, 75 insertions(+), 12 deletions(-) diff --git a/src/uu/shred/src/shred.rs b/src/uu/shred/src/shred.rs index f66a9dbc7fd..80b04e95d67 100644 --- a/src/uu/shred/src/shred.rs +++ b/src/uu/shred/src/shred.rs @@ -200,17 +200,22 @@ impl BytesWriter { } } - fn bytes_for_pass(&mut self, size: usize) -> &[u8] { + fn bytes_for_pass(&mut self, size: usize) -> Result<&[u8], std::io::Error> { match self { Self::Random { rng, buffer } => { let bytes = &mut buffer[..size]; - rng.fill(bytes); - bytes + match rng.try_fill(bytes) { + Ok(()) => Ok(bytes), + Err(err) => Err(std::io::Error::new( + std::io::ErrorKind::Other, + err.to_string(), + )), + } } Self::Pattern { offset, buffer } => { let bytes = &buffer[*offset..size + *offset]; *offset = (*offset + size) % PATTERN_LENGTH; - bytes + Ok(bytes) } } } @@ -237,8 +242,6 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { None => unreachable!(), }; - // TODO: implement --random-source - let remove_method = if matches.get_flag(options::WIPESYNC) { RemoveMethod::WipeSync } else if matches.contains_id(options::REMOVE) { @@ -504,6 +507,29 @@ fn wipe_file( None => metadata.len(), }; + // Random source errors + if let Some(random_source_str) = random_source { + let random_source_path = Path::new(random_source_str.as_str()); + if !random_source_path.exists() { + return Err(USimpleError::new( + 1, + format!( + "{}: No such file or directory", + random_source_path.maybe_quote() + ), + )); + } + if !random_source_path.is_file() { + return Err(USimpleError::new( + 1, + format!( + "{}: read error: Is a directory", + random_source_path.maybe_quote() + ), + )); + } + } + for (i, pass_type) in pass_sequence.into_iter().enumerate() { if verbose { let pass_name = pass_name(&pass_type); @@ -517,7 +543,7 @@ fn wipe_file( } // size is an optional argument for exactly how many bytes we want to shred // Ignore failed writes; just keep trying - show_if_err!(do_pass(&mut file, &pass_type, exact, size) + show_if_err!(do_pass(&mut file, &pass_type, exact, size, random_source) .map_err_context(|| format!("{}: File write pass failed", path.maybe_quote()))); } @@ -534,6 +560,7 @@ fn do_pass( pass_type: &PassType, exact: bool, file_size: u64, + random_source: &Option, ) -> Result<(), io::Error> { // We might be at the end of the file due to a previous iteration, so rewind. file.rewind()?; @@ -542,7 +569,15 @@ fn do_pass( // We start by writing BLOCK_SIZE times as many time as possible. for _ in 0..(file_size / BLOCK_SIZE as u64) { - let block = writer.bytes_for_pass(BLOCK_SIZE); + let data = writer.bytes_for_pass(BLOCK_SIZE); + let Ok(block) = data else { + let random_source_path = random_source + .clone() + .expect("random_source should be Some is None"); + let path = Path::new(&random_source_path); + show_error!("{}: end of file", path.maybe_quote()); + std::process::exit(1); + }; file.write_all(block)?; } @@ -551,7 +586,15 @@ fn do_pass( let bytes_left = (file_size % BLOCK_SIZE as u64) as usize; if bytes_left > 0 { let size = if exact { bytes_left } else { BLOCK_SIZE }; - let block = writer.bytes_for_pass(size); + let data = writer.bytes_for_pass(size); + let Ok(block) = data else { + let random_source_path = random_source + .clone() + .expect("random_source should be Some is None"); + let path = Path::new(&random_source_path); + show_error!("{}: end of file", path.maybe_quote()); + std::process::exit(1); + }; file.write_all(block)?; } diff --git a/src/uu/stdbuf/src/libstdbuf/build.rs b/src/uu/stdbuf/src/libstdbuf/build.rs index 6dcd6a86912..6fad9f33d66 100644 --- a/src/uu/stdbuf/src/libstdbuf/build.rs +++ b/src/uu/stdbuf/src/libstdbuf/build.rs @@ -7,5 +7,8 @@ use cpp_build::Config; fn main() { - Config::new().pic(true).build("src/libstdbuf.rs"); + Config::new() + .compiler("clang++") + .pic(true) + .build("src/libstdbuf.rs"); } diff --git a/tests/by-util/test_shred.rs b/tests/by-util/test_shred.rs index 6da1f1e3304..45129a11d2f 100644 --- a/tests/by-util/test_shred.rs +++ b/tests/by-util/test_shred.rs @@ -183,19 +183,36 @@ fn test_shred_empty() { #[test] fn test_random_source() { - let (at, mut ucmd) = at_and_ucmd!(); + let scene = TestScenario::new(util_name!()); + let at = &scene.fixtures; let file = "test_random_source"; let file_original_content = "test_shred file content"; at.write(file, file_original_content); + // File is missing + scene + .ucmd() + .arg("--random-source=test_random_file") + .arg(file) + .succeeds(); + // Write to random file let random_file = "test_random_file"; at.touch(random_file); + + scene + .ucmd() + .arg("--random-source=test_random_file") + .arg(file) + .succeeds(); + at.write(random_file, "random contents"); - ucmd.arg("--random-source=test_random_file") + scene + .ucmd() + .arg("--random-source=test_random_file") .arg(file) .succeeds();