Skip to content

Commit

Permalink
Revert "remove read_zipfile_from_stream()"
Browse files Browse the repository at this point in the history
This reverts commit 3abac52.
  • Loading branch information
cosmicexplorer committed Jul 17, 2024
1 parent c83e9b6 commit 3259c5e
Show file tree
Hide file tree
Showing 7 changed files with 197 additions and 71 deletions.
31 changes: 6 additions & 25 deletions benches/read_metadata.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,7 @@ use bencher::Bencher;
use getrandom::getrandom;
use tempdir::TempDir;
use zip::write::SimpleFileOptions;
use zip::{
result::ZipResult,
unstable::{
read::streaming::{StreamingZipEntry, ZipStreamFileMetadata},
stream::{ZipStreamReader, ZipStreamVisitor},
},
CompressionMethod, ZipArchive, ZipWriter,
};
use zip::{result::ZipResult, CompressionMethod, ZipArchive, ZipWriter};

const FILE_COUNT: usize = 15_000;
const FILE_SIZE: usize = 1024;
Expand Down Expand Up @@ -113,24 +106,12 @@ fn parse_stream_archive(bench: &mut Bencher) {
let out = dir.path().join("bench-out.zip");
fs::write(&out, &bytes).unwrap();

struct V;
impl ZipStreamVisitor for V {
fn visit_file(&mut self, _file: &mut StreamingZipEntry<impl Read>) -> ZipResult<()> {
Ok(())
}

fn visit_additional_metadata(
&mut self,
_metadata: &ZipStreamFileMetadata,
) -> ZipResult<()> {
Ok(())
}
}

bench.iter(|| {
let f = fs::File::open(&out).unwrap();
let archive = ZipStreamReader::new(f);
archive.visit(&mut V).unwrap();
let mut f = fs::File::open(&out).unwrap();
while zip::read::read_zipfile_from_stream(&mut f)
.unwrap()
.is_some()
{}
});
bench.bytes = bytes.len() as u64;
}
Expand Down
7 changes: 2 additions & 5 deletions examples/stdin_info.rs
Original file line number Diff line number Diff line change
@@ -1,19 +1,16 @@
use std::io::{self, Read};

use zip::unstable::read::{streaming::StreamingArchive, ArchiveEntry};

fn main() {
std::process::exit(real_main());
}

fn real_main() -> i32 {
let stdin = io::stdin();
let stdin_handle = stdin.lock();
let mut stdin_handle = stdin.lock();
let mut buf = [0u8; 16];

let mut archive = StreamingArchive::new(stdin_handle);
loop {
match archive.next_entry() {
match zip::read::read_zipfile_from_stream(&mut stdin_handle) {
Ok(Some(mut file)) => {
println!(
"{}: {} bytes ({} bytes packed)",
Expand Down
24 changes: 4 additions & 20 deletions fuzz/fuzz_targets/fuzz_read.rs
100755 → 100644
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,7 @@
use libfuzzer_sys::fuzz_target;
use std::io::{Read, Seek, SeekFrom};
use tikv_jemallocator::Jemalloc;
use zip::unstable::{
read::streaming::{StreamingArchive, StreamingZipEntry, ZipStreamFileMetadata},
stream::ZipStreamVisitor,
};
use zip::read::read_zipfile_from_stream;

const MAX_BYTES_TO_READ: u64 = 1 << 24;

Expand All @@ -21,24 +18,11 @@ fn decompress_all(data: &[u8]) -> Result<(), Box<dyn std::error::Error>> {
let mut file = zip.by_index(i)?.take(MAX_BYTES_TO_READ);
std::io::copy(&mut file, &mut std::io::sink())?;
}

let mut reader = zip.into_inner();
reader.rewind()?;

struct V;
impl ZipStreamVisitor for V {
fn visit_file(&mut self, file: &mut StreamingZipEntry<impl Read>) -> ZipResult<()> {
std::io::copy(&mut file, &mut std::io::sink())?
}

fn visit_additional_metadata(&mut self, metadata: &ZipStreamFileMetadata) -> ZipResult<()> {
Ok(())
}
reader.seek(SeekFrom::Start(0))?;
while let Ok(Some(mut file)) = read_zipfile_from_stream(&mut reader) {
std::io::copy(&mut file, &mut std::io::sink())?;
}

let archive = StreamingArchive::new(reader)?;
archive.visit(&mut V)?;

Ok(())
}

Expand Down
76 changes: 72 additions & 4 deletions src/read.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1900,6 +1900,71 @@ impl<'a> Drop for ZipFile<'a> {
}
}

/// Read ZipFile structures from a non-seekable reader.
///
/// This is an alternative method to read a zip file. If possible, use the ZipArchive functions
/// as some information will be missing when reading this manner.
///
/// Reads a file header from the start of the stream. Will return `Ok(Some(..))` if a file is
/// present at the start of the stream. Returns `Ok(None)` if the start of the central directory
/// is encountered. No more files should be read after this.
///
/// The Drop implementation of ZipFile ensures that the reader will be correctly positioned after
/// the structure is done.
///
/// Missing fields are:
/// * `comment`: set to an empty string
/// * `data_start`: set to 0
/// * `external_attributes`: `unix_mode()`: will return None
pub fn read_zipfile_from_stream<'a, R: Read>(reader: &'a mut R) -> ZipResult<Option<ZipFile<'_>>> {
// We can't use the typical ::parse() method, as we follow separate code paths depending on the
// "magic" value (since the magic value will be from the central directory header if we've
// finished iterating over all the actual files).
/* TODO: smallvec? */
let mut block = [0u8; mem::size_of::<ZipLocalEntryBlock>()];
reader.read_exact(&mut block)?;
let block: Box<[u8]> = block.into();

let signature = spec::Magic::from_first_le_bytes(&block);

match signature {
spec::Magic::LOCAL_FILE_HEADER_SIGNATURE => (),
spec::Magic::CENTRAL_DIRECTORY_HEADER_SIGNATURE => return Ok(None),
_ => return Err(ZipLocalEntryBlock::WRONG_MAGIC_ERROR),
}

let block = ZipLocalEntryBlock::interpret(&block)?;

let mut result = ZipFileData::from_local_block(block, reader)?;

match parse_extra_field(&mut result) {
Ok(..) | Err(ZipError::Io(..)) => {}
Err(e) => return Err(e),
}

let limit_reader = (reader as &'a mut dyn Read).take(result.compressed_size);

let result_crc32 = result.crc32;
let result_compression_method = result.compression_method;
let crypto_reader = make_crypto_reader(
result_compression_method,
result_crc32,
result.last_modified_time,
result.using_data_descriptor,
limit_reader,
None,
None,
#[cfg(feature = "aes-crypto")]
result.compressed_size,
)?;

Ok(Some(ZipFile {
data: Cow::Owned(result),
crypto_reader: None,
reader: make_reader(result_compression_method, result_crc32, crypto_reader)?,
}))
}

#[cfg(test)]
mod test {
use crate::result::ZipResult;
Expand Down Expand Up @@ -1952,13 +2017,16 @@ mod test {

#[test]
fn zip_read_streaming() {
use crate::unstable::read::streaming::StreamingArchive;
use super::read_zipfile_from_stream;

let mut v = Vec::new();
v.extend_from_slice(include_bytes!("../tests/data/mimetype.zip"));
let reader = Cursor::new(v);
let mut archive = StreamingArchive::new(reader);
while archive.next_entry().unwrap().is_some() {}
let mut reader = Cursor::new(v);
loop {
if read_zipfile_from_stream(&mut reader).unwrap().is_none() {
break;
}
}
}

#[test]
Expand Down
Loading

0 comments on commit 3259c5e

Please sign in to comment.