diff --git a/storage/src/device.rs b/storage/src/device.rs index 6c559cf481c..5151169e058 100644 --- a/storage/src/device.rs +++ b/storage/src/device.rs @@ -1398,6 +1398,8 @@ pub mod v5 { #[cfg(test)] mod tests { + use std::path::PathBuf; + use super::*; use crate::test::MockChunkInfo; @@ -1611,4 +1613,32 @@ mod tests { assert_eq!(size, iovec.size()); assert_eq!(chunk_count, iovec.len() as u32); } + + #[test] + fn test_blob_info_blob_meta_id() { + let blob_info = BlobInfo::new( + 1, + "blob_id".to_owned(), + 0, + 0, + 0, + 1, + BlobFeatures::SEPARATE | BlobFeatures::INLINED_FS_META, + ); + + let root_dir = &std::env::var("CARGO_MANIFEST_DIR").expect("$CARGO_MANIFEST_DIR"); + let mut source_path = PathBuf::from(root_dir); + source_path.push("../tests/texture/blobs/be7d77eeb719f70884758d1aa800ed0fb09d701aaec469964e9d54325f0d5fef"); + + assert!(blob_info + .set_blob_id_from_meta_path(source_path.as_path()) + .is_ok()); + + let id = blob_info.get_blob_meta_id(); + assert!(id.is_ok()); + assert_eq!( + id.unwrap(), + "be7d77eeb719f70884758d1aa800ed0fb09d701aaec469964e9d54325f0d5fef".to_owned() + ); + } } diff --git a/storage/src/meta/batch.rs b/storage/src/meta/batch.rs index fefdc056d9b..0f98326146c 100644 --- a/storage/src/meta/batch.rs +++ b/storage/src/meta/batch.rs @@ -153,3 +153,62 @@ impl BatchContextGenerator { Ok((data, self.contexts.len() as u32)) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_batch_inflate_context() { + let mut ctx = BatchInflateContext { + compressed_offset: 0, + compressed_size: 0, + uncompressed_batch_size: 0, + __reserved1: 0, + __reserved2: 0, + __reserved3: 0, + }; + ctx.set_compressed_offset(0x10); + assert_eq!(ctx.compressed_offset(), 0x10); + ctx.set_compressed_size(0x20); + assert_eq!(ctx.compressed_size(), 0x20); + assert_eq!(ctx.compressed_end(), 0x30); + let mut v = [0u8; 40]; + v[0] = 0x10; + v[8] = 0x20; + assert_eq!(ctx.as_slice(), v); + } + + #[test] + fn test_batch_context_generator() { + let mut generator = BatchContextGenerator { + chunk_data_buf: vec![1u8, 2, 3, 4, 5], + contexts: vec![], + }; + assert!(!generator.chunk_data_buf_is_empty()); + let data = [6u8, 7, 8, 9]; + generator.append_chunk_data_buf(&data); + assert_eq!( + generator.chunk_data_buf_len(), + generator.chunk_data_buf().len() + ); + + generator.add_context(0x10, 0x20); + assert_eq!(generator.contexts.len(), 1); + + let mut data = vec![0u8; 40]; + data[0] = 0x10; + data[8] = 0x20; + data[12] = 9; + assert_eq!(generator.to_vec().unwrap(), (data, 1 as u32)); + + let ctx = BatchContextGenerator::new(8); + assert!(ctx.is_ok()); + assert_eq!(ctx.unwrap().chunk_data_buf().capacity(), 8); + + assert!(generator.generate_chunk_info(0, 2, true).is_ok()); + + generator.clear_chunk_data_buf(); + assert_eq!(generator.chunk_data_buf_len(), 0); + } +} diff --git a/storage/src/meta/chunk_info_v2.rs b/storage/src/meta/chunk_info_v2.rs index 9a2ce255a9f..f19d9477466 100644 --- a/storage/src/meta/chunk_info_v2.rs +++ b/storage/src/meta/chunk_info_v2.rs @@ -306,6 +306,36 @@ mod tests { chunk.set_zran_offset(5); assert_eq!(chunk.get_zran_index(), 3); assert_eq!(chunk.get_zran_offset(), 5); + chunk.set_zran(false); + assert!(!chunk.is_zran()); + + let before = chunk.uncomp_info; + chunk.set_compressed(true); + chunk.set_compressed(false); + assert_eq!(chunk.uncomp_info as u64, before); + + chunk.set_encrypted(true); + assert!(chunk.is_encrypted()); + + let before = chunk.uncomp_info; + chunk.set_batch(true); + chunk.set_batch(false); + assert_eq!(chunk.uncomp_info as u64, before); + + chunk.set_data(0x10); + assert_eq!(chunk.data as u64, 0x10); + + chunk.set_batch(true); + chunk.set_batch_index(0x20); + assert_eq!(chunk.data as u64, 137438953488); + + chunk.set_uncompressed_offset_in_batch_buf(0x30); + assert_eq!(chunk.data as u64, 137438953520); + + assert_eq!(chunk.flags(), 12); + assert_eq!(chunk.get_batch_index(), 32); + assert_eq!(chunk.get_uncompressed_offset_in_batch_buf(), 48); + assert_eq!(chunk.get_data(), 137438953520); // For testing old format compatibility. let chunk = BlobChunkInfoV2Ondisk { @@ -380,4 +410,33 @@ mod tests { .get_chunk_index_nocheck(&state, 0x102000, false) .unwrap_err(); } + + #[test] + fn test_chunk_on_disk_validate() { + let mut ctx = BlobCompressionContext::default(); + let mut chunk = BlobChunkInfoV2Ondisk::default(); + println!("{}", chunk); + + chunk.set_compressed_offset(0x10); + chunk.set_compressed_size(0x20); + chunk.set_encrypted(false); + chunk.set_compressed(false); + chunk.set_uncompressed_size(0x30); + chunk.set_compressed_size(0x40); + chunk.set_zran(true); + ctx.compressed_size = 0x100; + ctx.uncompressed_size = 0x40; + ctx.blob_features = 0; + assert!(chunk.validate(&ctx).is_err()); + + chunk.set_encrypted(true); + assert!(chunk.validate(&ctx).is_err()); + + ctx.blob_features = BlobFeatures::ZRAN.bits(); + chunk.set_zran_index(0); + assert!(chunk.validate(&ctx).is_err()); + + chunk.set_zran(false); + assert!(chunk.validate(&ctx).is_ok()); + } } diff --git a/storage/src/meta/mod.rs b/storage/src/meta/mod.rs index 5787f5bcf94..eff935cda17 100644 --- a/storage/src/meta/mod.rs +++ b/storage/src/meta/mod.rs @@ -1997,6 +1997,7 @@ pub(crate) mod tests { use crate::device::BlobFeatures; use crate::RAFS_DEFAULT_CHUNK_SIZE; use nix::sys::uio; + use nydus_utils::digest::{self, DigestHasher}; use nydus_utils::metrics::BackendMetrics; use std::fs::File; use std::os::unix::io::AsRawFd; @@ -2202,4 +2203,110 @@ pub(crate) mod tests { .get_chunks_compressed(0x1000000, 0x1, RAFS_DEFAULT_CHUNK_SIZE, false) .is_err()); } + + #[test] + fn test_blob_compression_context_header_getters_and_setters() { + let mut header = BlobCompressionContextHeader::default(); + + assert_eq!(header.features(), 0); + header.set_aligned(true); + assert!(header.is_4k_aligned()); + header.set_aligned(false); + + header.set_inlined_fs_meta(true); + assert!(header.has_feature(BlobFeatures::INLINED_FS_META)); + header.set_inlined_fs_meta(false); + + header.set_chunk_info_v2(true); + assert!(header.has_feature(BlobFeatures::CHUNK_INFO_V2)); + header.set_chunk_info_v2(false); + + header.set_ci_zran(true); + assert!(header.has_feature(BlobFeatures::ZRAN)); + header.set_ci_zran(false); + + header.set_separate_blob(true); + assert!(header.has_feature(BlobFeatures::SEPARATE)); + header.set_separate_blob(false); + + header.set_ci_batch(true); + assert!(header.has_feature(BlobFeatures::BATCH)); + header.set_ci_batch(false); + + header.set_inlined_chunk_digest(true); + assert!(header.has_feature(BlobFeatures::INLINED_CHUNK_DIGEST)); + header.set_inlined_chunk_digest(false); + + header.set_has_tar_header(true); + assert!(header.has_feature(BlobFeatures::HAS_TAR_HEADER)); + header.set_has_tar_header(false); + + header.set_has_toc(true); + assert!(header.has_feature(BlobFeatures::HAS_TOC)); + header.set_has_toc(false); + + header.set_cap_tar_toc(true); + assert!(header.has_feature(BlobFeatures::CAP_TAR_TOC)); + header.set_cap_tar_toc(false); + + header.set_tarfs(true); + assert!(header.has_feature(BlobFeatures::TARFS)); + header.set_tarfs(false); + + header.set_encrypted(true); + assert!(header.has_feature(BlobFeatures::ENCRYPTED)); + header.set_encrypted(false); + + assert_eq!(header.features(), 0); + + assert_eq!(header.ci_compressor(), compress::Algorithm::Lz4Block); + header.set_ci_compressor(compress::Algorithm::GZip); + assert_eq!(header.ci_compressor(), compress::Algorithm::GZip); + header.set_ci_compressor(compress::Algorithm::Zstd); + assert_eq!(header.ci_compressor(), compress::Algorithm::Zstd); + + let mut hasher = RafsDigest::hasher(digest::Algorithm::Sha256); + hasher.digest_update(header.as_bytes()); + let hash: String = hasher.digest_finalize().into(); + assert_eq!( + hash, + String::from("f56a1129d3df9fc7d60b26dbf495a60bda3dfc265f4f37854e4a36b826b660fc") + ); + + assert_eq!(header.ci_entries(), 0); + header.set_ci_entries(1); + assert_eq!(header.ci_entries(), 1); + + assert_eq!(header.ci_compressed_offset(), 0); + header.set_ci_compressed_offset(1); + assert_eq!(header.ci_compressed_offset(), 1); + + assert_eq!(header.ci_compressed_size(), 0); + header.set_ci_compressed_size(1); + assert_eq!(header.ci_compressed_size(), 1); + + assert_eq!(header.ci_uncompressed_size(), 0); + header.set_ci_uncompressed_size(1); + assert_eq!(header.ci_uncompressed_size(), 1); + + assert_eq!(header.ci_zran_count(), 0); + header.set_ci_zran_count(1); + assert_eq!(header.ci_zran_count(), 1); + + assert_eq!(header.ci_zran_offset(), 0); + header.set_ci_zran_offset(1); + assert_eq!(header.ci_zran_offset(), 1); + + assert_eq!(header.ci_zran_size(), 0); + header.set_ci_zran_size(1); + assert_eq!(header.ci_zran_size(), 1); + } + + #[test] + fn test_format_blob_features() { + let features = !BlobFeatures::default(); + let content = format_blob_features(features); + assert!(content.contains("aligned")); + assert!(content.contains("fs-meta")); + } } diff --git a/storage/src/meta/toc.rs b/storage/src/meta/toc.rs index bc64026ffcf..91fc8ea2601 100644 --- a/storage/src/meta/toc.rs +++ b/storage/src/meta/toc.rs @@ -778,7 +778,7 @@ mod tests { let blob_mgr = BlobFactory::new_backend(&config, id).unwrap(); let blob = blob_mgr.get_reader(id).unwrap(); let location = TocLocation::with_digest(9010, 1024, digest); - let list = + let mut list = TocEntryList::read_from_blob::(blob.as_ref(), None, &location).unwrap(); assert_eq!(list.entries.len(), 4); @@ -789,15 +789,27 @@ mod tests { let mut buf = Vec::new(); let entry = list.get_entry(TOC_ENTRY_BLOB_META).unwrap(); - assert_eq!(entry.uncompressed_size, 0x30); + assert_eq!(entry.uncompressed_size(), 0x30); entry.extract_from_reader(blob.clone(), &mut buf).unwrap(); assert!(!buf.is_empty()); let mut buf = Vec::new(); let entry = list.get_entry(TOC_ENTRY_BLOB_META_HEADER).unwrap(); - assert_eq!(entry.uncompressed_size, 0x1000); + assert_eq!(entry.uncompressed_size(), 0x1000); entry.extract_from_reader(blob.clone(), &mut buf).unwrap(); assert!(!buf.is_empty()); + + assert!(list + .add( + TOC_ENTRY_BLOB_DIGEST, + compress::Algorithm::Lz4Block, + digest, + 0, + 2, + 3 + ) + .is_ok()); + assert!(list.get_entry(TOC_ENTRY_BLOB_DIGEST).is_some()); } #[test] @@ -892,4 +904,92 @@ mod tests { assert_eq!(flags, TocEntryFlags::COMPRESSION_ZSTD); let _e = TocEntryFlags::try_from(compress::Algorithm::GZip).unwrap_err(); } + + fn extract_from_buf_with_different_flags(entry: &TocEntry, buf: &[u8]) -> Result { + let tmp_file = TempFile::new(); + let mut file = OpenOptions::new() + .write(true) + .read(true) + .open(tmp_file.unwrap().as_path()) + .unwrap(); + + entry.extract_from_buf(&buf, &mut file)?; + + let mut hasher = RafsDigest::hasher(digest::Algorithm::Sha256); + let mut buffer = [0; 1024]; + loop { + let count = file.read(&mut buffer)?; + if count == 0 { + break; + } + hasher.digest_update(&buffer[..count]); + } + Ok(hasher.digest_finalize().into()) + } + + #[test] + fn test_extract_from_buf() { + let mut entry = TocEntry { + flags: 0, + reserved1: 0, + name: [0u8; 16], + uncompressed_digest: [ + 45, 15, 227, 154, 167, 87, 190, 28, 152, 93, 55, 27, 96, 217, 56, 121, 96, 131, + 226, 94, 70, 74, 193, 156, 222, 228, 46, 156, 49, 169, 143, 53, + ], + compressed_offset: 0, + compressed_size: 0, + uncompressed_size: 0, + reserved2: [0u8; 48], + }; + + let buf = [ + 79u8, 223, 187, 54, 239, 116, 163, 198, 58, 40, 226, 171, 175, 165, 64, 68, 199, 89, + 65, 85, 190, 182, 221, 173, 159, 54, 130, 92, 254, 88, 40, 108, + ]; + + entry.flags = TocEntryFlags::COMPRESSION_LZ4_BLOCK.bits(); + assert!(extract_from_buf_with_different_flags(&entry, &buf).is_err()); + + entry.flags = (!TocEntryFlags::empty()).bits() + 1; + assert!(extract_from_buf_with_different_flags(&entry, &buf).is_err()); + + entry.flags = TocEntryFlags::COMPRESSION_NONE.bits(); + assert!(extract_from_buf_with_different_flags(&entry, &buf).is_err()); + entry.uncompressed_size = 32; + let s = extract_from_buf_with_different_flags(&entry, &buf); + assert!(s.is_ok()); + assert_eq!( + s.unwrap(), + String::from("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") + ); + + let root_dir = &std::env::var("CARGO_MANIFEST_DIR").expect("$CARGO_MANIFEST_DIR"); + let path = Path::new(root_dir) + .join("../tests/texture/zstd") + .join("2fa78cad554b75ac91a4a125ed148d0ddeb25efa4aaa8bd80e5dc292690a4dca.zst"); + let mut file = OpenOptions::new().read(true).open(path.as_path()).unwrap(); + let mut buffer = [0; 1024]; + let mut buf = vec![]; + loop { + let count = file.read(&mut buffer).unwrap(); + if count == 0 { + break; + } + buf.extend_from_slice(&buffer[..count]); + } + entry.flags = TocEntryFlags::COMPRESSION_ZSTD.bits(); + entry.uncompressed_size = 10034; + assert!(extract_from_buf_with_different_flags(&entry, &buf).is_err()); + entry.uncompressed_digest = [ + 47, 167, 140, 173, 85, 75, 117, 172, 145, 164, 161, 37, 237, 20, 141, 13, 222, 178, 94, + 250, 74, 170, 139, 216, 14, 93, 194, 146, 105, 10, 77, 202, + ]; + let s = extract_from_buf_with_different_flags(&entry, &buf); + assert!(s.is_ok()); + assert_eq!( + s.unwrap(), + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855".to_owned() + ); + } } diff --git a/tests/texture/zstd/2fa78cad554b75ac91a4a125ed148d0ddeb25efa4aaa8bd80e5dc292690a4dca.zst b/tests/texture/zstd/2fa78cad554b75ac91a4a125ed148d0ddeb25efa4aaa8bd80e5dc292690a4dca.zst new file mode 100644 index 00000000000..6adeda93153 Binary files /dev/null and b/tests/texture/zstd/2fa78cad554b75ac91a4a125ed148d0ddeb25efa4aaa8bd80e5dc292690a4dca.zst differ