Skip to content

Commit

Permalink
downgrade ZIP dependency to a version that supports >65k files
Browse files Browse the repository at this point in the history
  • Loading branch information
syphar committed Jul 11, 2024
1 parent 2eb0516 commit 7ad8c62
Show file tree
Hide file tree
Showing 3 changed files with 94 additions and 21 deletions.
70 changes: 60 additions & 10 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ font-awesome-as-a-crate = { path = "crates/font-awesome-as-a-crate" }
dashmap = "5.1.0"
string_cache = "0.8.0"
postgres-types = { version = "0.2", features = ["derive"] }
zip = {version = "2.1.3", default-features = false, features = ["bzip2"]}
zip = {version = "1.1.4", default-features = false, features = ["bzip2"]}
bzip2 = "0.4.4"
getrandom = "0.2.1"
itertools = { version = "0.13.0", optional = true}
Expand Down
43 changes: 33 additions & 10 deletions src/storage/archive_index.rs
Original file line number Diff line number Diff line change
Expand Up @@ -119,31 +119,33 @@ mod tests {
use std::io::Write;
use zip::write::SimpleFileOptions;

fn create_test_archive() -> fs::File {
fn create_test_archive(file_count: u32) -> fs::File {
let mut tf = tempfile::tempfile().unwrap();

let objectcontent: Vec<u8> = (0..255).collect();

let mut archive = zip::ZipWriter::new(tf);
archive
.start_file(
"testfile1",
SimpleFileOptions::default().compression_method(zip::CompressionMethod::Bzip2),
)
.unwrap();
archive.write_all(&objectcontent).unwrap();
for i in 0..file_count {
archive
.start_file(
format!("testfile{i}"),
SimpleFileOptions::default().compression_method(zip::CompressionMethod::Bzip2),
)
.unwrap();
archive.write_all(&objectcontent).unwrap();
}
tf = archive.finish().unwrap();
tf
}

#[test]
fn index_create_save_load_sqlite() {
let mut tf = create_test_archive();
let mut tf = create_test_archive(1);

let tempfile = tempfile::NamedTempFile::new().unwrap().into_temp_path();
create(&mut tf, &tempfile).unwrap();

let fi = find_in_file(&tempfile, "testfile1").unwrap().unwrap();
let fi = find_in_file(&tempfile, "testfile0").unwrap().unwrap();

assert_eq!(fi.range, FileRange::new(39, 459));
assert_eq!(fi.compression, CompressionAlgorithm::Bzip2);
Expand All @@ -152,4 +154,25 @@ mod tests {
.unwrap()
.is_none());
}

#[test]
fn archive_with_more_than_65k_files() {
let mut tf = create_test_archive(100_000);

let tempfile = tempfile::NamedTempFile::new().unwrap().into_temp_path();
create(&mut tf, &tempfile).unwrap();

let connection = Connection::open_with_flags(
tempfile,
OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
)
.unwrap();
let mut stmt = connection.prepare("SELECT count(*) FROM files").unwrap();

let count = stmt
.query_row([], |row| Ok(row.get::<_, usize>(0)))
.unwrap()
.unwrap();
assert_eq!(count, 100_000);
}
}

0 comments on commit 7ad8c62

Please sign in to comment.