From 1359c32d657b694ec07daaa18e9e261eec04a2ae Mon Sep 17 00:00:00 2001 From: Danny McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Wed, 23 Aug 2023 19:08:08 -0400 Subject: [PATCH] fix waiting to link binaries, but tensorflow example is still huge! --- lib/benches/my_benchmark.rs | 88 ++++++++++++++++++------------------- lib/src/zip.rs | 7 ++- 2 files changed, 49 insertions(+), 46 deletions(-) diff --git a/lib/benches/my_benchmark.rs b/lib/benches/my_benchmark.rs index 2103a50..ff3f9b3 100644 --- a/lib/benches/my_benchmark.rs +++ b/lib/benches/my_benchmark.rs @@ -12,38 +12,30 @@ use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Through mod parallel_merge { use super::*; - use libmedusa_zip::{self as lib, destination::OutputWrapper}; + use libmedusa_zip as lib; use tempfile; use tokio::runtime::Runtime; use zip::{self, result::ZipError}; - use std::{fs, io, time::Duration}; + use std::{fs, path::Path, time::Duration}; - /* This file is 461M, or about half a gigabyte, with multiple individual very - * large binary files. */ - /* const LARGE_ZIP_CONTENTS: &'static [u8] = */ - /* include_bytes!("tensorflow_gpu-2.5.3-cp38-cp38-manylinux2010_x86_64.whl"); */ - - /* This file is 37K. */ - const SMALLER_ZIP_CONTENTS: &'static [u8] = include_bytes!("Keras-2.4.3-py2.py3-none-any.whl"); - - fn prepare_memory_zip( - zip_contents: &[u8], + fn extract_example_zip( + target: &Path, ) -> Result<(Vec, tempfile::TempDir), ZipError> { /* Create the temp dir to extract into. */ let extract_dir = tempfile::tempdir()?; - /* Load the zip archive from memory. */ - let reader = io::Cursor::new(zip_contents); - let mut large_zip = zip::ZipArchive::new(reader)?; + /* Load the zip archive from file. */ + let handle = fs::OpenOptions::new().read(true).open(target)?; + let mut zip_archive = zip::ZipArchive::new(handle)?; /* Extract the zip's contents. */ - large_zip.extract(extract_dir.path())?; + zip_archive.extract(extract_dir.path())?; /* Generate the input to a MedusaZip by associating the (relative) file names * from the zip to their (absolute) extracted output paths. */ - let input_files: Vec = large_zip.file_names() + let input_files: Vec = zip_archive.file_names() /* Ignore any directories, which are not represented in FileSource structs. */ .filter(|f| !f.ends_with('/')) .map(|f| { @@ -59,7 +51,6 @@ mod parallel_merge { Ok((input_files, extract_dir)) } - async fn create_basic_zip( input_files: Vec, parallelism: lib::zip::Parallelism, @@ -70,12 +61,12 @@ mod parallel_merge { modifications: lib::zip::EntryModifications::default(), parallelism, }; - let output_zip = OutputWrapper::wrap(zip::ZipWriter::new(tempfile::tempfile()?)); + let output_zip = + lib::destination::OutputWrapper::wrap(zip::ZipWriter::new(tempfile::tempfile()?)); let mut output_zip = zip_spec.zip(output_zip).await?.reclaim(); Ok(output_zip.finish_into_readable()?) } - pub fn bench_zips(c: &mut Criterion) { let rt = Runtime::new().unwrap(); @@ -89,48 +80,55 @@ mod parallel_merge { * some small true changes. */ .significance_level(0.01); - for (id, zip_contents, n, t) in [ + for (filename, n, t) in [ ( + /* This file is 37K. */ "Keras-2.4.3-py2.py3-none-any.whl", - SMALLER_ZIP_CONTENTS, 1000, Duration::from_secs(7), ), - /* ("tensorflow_gpu-2.5.3-cp38-cp38-manylinux2010_x86_64.whl", LARGE_ZIP_CONTENTS), */ + ( + /* This file is 461M, or about half a gigabyte, with multiple individual very + * large binary files. */ + "tensorflow_gpu-2.5.3-cp38-cp38-manylinux2010_x86_64.whl", + 10, + Duration::from_secs(330), + ), ] .iter() { + let target = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("benches") + .join(filename); + let zip_len = target.metadata().unwrap().len(); + + let id = format!("{}({} bytes)", filename, zip_len); + group .sample_size(*n) .measurement_time(*t) - .throughput(Throughput::Bytes(zip_contents.len() as u64)); + .throughput(Throughput::Bytes(zip_len as u64)); /* FIXME: assigning `_` to the second arg of this tuple will destroy the * extract dir, which is only a silent error producing an empty file!!! * AWFUL UX!!! */ - let (input_files, _tmp_extract_dir) = prepare_memory_zip(zip_contents).unwrap(); - group.noise_threshold(0.03); - group.bench_with_input( - BenchmarkId::new(*id, "ParallelMerge"), - &lib::zip::Parallelism::ParallelMerge, - |b, p| { + let (input_files, _tmp_extract_dir) = extract_example_zip(&target).unwrap(); + + for (noise_threshold, parallelism) in [ + (0.03, lib::zip::Parallelism::ParallelMerge), + /* This says we don't care about improvements under 5%, which is a huge gap, + * but otherwise the synchronous keras benchmark will report e.g. 4.8% + * improvement immediately after the last bench. */ + (0.05, lib::zip::Parallelism::Synchronous), + ] + .iter() + { + group.noise_threshold(*noise_threshold); + group.bench_with_input(BenchmarkId::new(&id, parallelism), parallelism, |b, p| { b.to_async(&rt) .iter(|| create_basic_zip(input_files.clone(), *p)); - }, - ); - - /* This says we don't care about improvements under 5%, which is a huge gap, - * but otherwise the synchronous keras benchmark will report e.g. 4.8% - * improvement immediately after the last bench. */ - group.noise_threshold(0.05); - group.bench_with_input( - BenchmarkId::new(*id, "Synchronous"), - &lib::zip::Parallelism::Synchronous, - |b, p| { - b.to_async(&rt) - .iter(|| create_basic_zip(input_files.clone(), *p)); - }, - ); + }); + } } group.finish(); diff --git a/lib/src/zip.rs b/lib/src/zip.rs index 9bccec9..dec2223 100644 --- a/lib/src/zip.rs +++ b/lib/src/zip.rs @@ -579,10 +579,15 @@ impl IntermediateSingleEntry { } } -#[derive(Copy, Clone, Default, Debug)] +#[derive(Copy, Clone, Default, Debug, Display)] +#[ignore_extra_doc_attributes] pub enum Parallelism { + /// + /// /// Read source files and copy them to the output zip in order. Synchronous, + /// + /// /// Parallelize creation by splitting up the input into chunks. #[default] ParallelMerge,