From 77cdee9b0bf245034549137aa19e1d5c9f98f786 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 9 Dec 2024 23:05:02 +0100 Subject: [PATCH] feat: extract zstd compressors --- Cargo.lock | 9 +- Cargo.toml | 2 + crates/primitives/Cargo.toml | 7 +- crates/primitives/src/lib.rs | 4 - crates/primitives/src/receipt.rs | 4 +- crates/primitives/src/transaction/mod.rs | 8 +- crates/storage/zstd-compressors/Cargo.toml | 19 ++++ .../zstd-compressors}/receipt_dictionary.bin | Bin .../zstd-compressors/src/lib.rs} | 82 +++++++++++------- .../transaction_dictionary.bin | Bin 10 files changed, 90 insertions(+), 45 deletions(-) create mode 100644 crates/storage/zstd-compressors/Cargo.toml rename crates/{primitives/src/compression => storage/zstd-compressors}/receipt_dictionary.bin (100%) rename crates/{primitives/src/compression/mod.rs => storage/zstd-compressors/src/lib.rs} (62%) rename crates/{primitives/src/compression => storage/zstd-compressors}/transaction_dictionary.bin (100%) diff --git a/Cargo.lock b/Cargo.lock index 58982032e30f..e34d2dce92a4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8743,6 +8743,7 @@ dependencies = [ "reth-static-file-types", "reth-testing-utils", "reth-trie-common", + "reth-zstd-compressors", "revm-primitives", "rstest", "secp256k1", @@ -8750,7 +8751,6 @@ dependencies = [ "serde_json", "serde_with", "test-fuzz", - "zstd", ] [[package]] @@ -9647,6 +9647,13 @@ dependencies = [ "thiserror 2.0.5", ] +[[package]] +name = "reth-zstd-compressors" +version = "1.1.2" +dependencies = [ + "zstd", +] + [[package]] name = "revm" version = "18.0.0" diff --git a/Cargo.toml b/Cargo.toml index 142b00290b9a..beef7d330dd4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -118,6 +118,7 @@ members = [ "crates/storage/nippy-jar/", "crates/storage/provider/", "crates/storage/storage-api/", + "crates/storage/zstd-compressors/", "crates/tasks/", "crates/tokio-util/", "crates/tracing/", @@ -422,6 +423,7 @@ reth-trie-common = { path = "crates/trie/common" } reth-trie-db = { path = "crates/trie/db" } reth-trie-parallel = { path = "crates/trie/parallel" } reth-trie-sparse = { path = "crates/trie/sparse" } +reth-zstd-compressors = { path = "crates/storage/zstd-compressors", default-features = false } # revm revm = { version = "18.0.0", features = ["std"], default-features = false } diff --git a/crates/primitives/Cargo.toml b/crates/primitives/Cargo.toml index 2f8f37bcd356..109b20ec2bcc 100644 --- a/crates/primitives/Cargo.toml +++ b/crates/primitives/Cargo.toml @@ -18,6 +18,7 @@ reth-ethereum-forks.workspace = true reth-static-file-types.workspace = true revm-primitives = { workspace = true, features = ["serde"] } reth-codecs = { workspace = true, optional = true } +reth-zstd-compressors = { workspace = true, optional = true } # ethereum alloy-consensus.workspace = true @@ -55,7 +56,6 @@ rand = { workspace = true, optional = true } rayon.workspace = true serde.workspace = true serde_with = { workspace = true, optional = true } -zstd = { workspace = true, features = ["experimental"], optional = true } # arbitrary utils arbitrary = { workspace = true, features = ["derive"], optional = true } @@ -108,11 +108,12 @@ std = [ "alloy-rlp/std", "reth-ethereum-forks/std", "bytes/std", - "derive_more/std" + "derive_more/std", + "reth-zstd-compressors?/std" ] reth-codec = [ "dep:reth-codecs", - "dep:zstd", + "dep:reth-zstd-compressors", "dep:modular-bitfield", "std", "reth-primitives-traits/reth-codec", ] diff --git a/crates/primitives/src/lib.rs b/crates/primitives/src/lib.rs index edbc73a9362d..18fe1498b8a8 100644 --- a/crates/primitives/src/lib.rs +++ b/crates/primitives/src/lib.rs @@ -27,8 +27,6 @@ pub use traits::*; #[cfg(feature = "alloy-compat")] mod alloy_compat; mod block; -#[cfg(feature = "reth-codec")] -mod compression; pub mod proofs; mod receipt; pub use reth_static_file_types as static_file; @@ -38,8 +36,6 @@ pub use block::{generate_valid_header, valid_header_strategy}; pub use block::{ Block, BlockBody, BlockWithSenders, SealedBlock, SealedBlockFor, SealedBlockWithSenders, }; -#[cfg(feature = "reth-codec")] -pub use compression::*; pub use receipt::{gas_spent_by_transactions, Receipt, Receipts}; pub use reth_primitives_traits::{ logs_bloom, Account, Bytecode, GotExpected, GotExpectedBoxed, Header, HeaderError, Log, diff --git a/crates/primitives/src/receipt.rs b/crates/primitives/src/receipt.rs index 419c36c2080b..62c664e22a46 100644 --- a/crates/primitives/src/receipt.rs +++ b/crates/primitives/src/receipt.rs @@ -12,9 +12,9 @@ use derive_more::{DerefMut, From, IntoIterator}; use reth_primitives_traits::receipt::ReceiptExt; use serde::{Deserialize, Serialize}; -#[cfg(feature = "reth-codec")] -use crate::compression::{RECEIPT_COMPRESSOR, RECEIPT_DECOMPRESSOR}; use crate::TxType; +#[cfg(feature = "reth-codec")] +use reth_zstd_compressors::{RECEIPT_COMPRESSOR, RECEIPT_DECOMPRESSOR}; /// Retrieves gas spent by transactions as a vector of tuples (transaction index, gas used). pub use reth_primitives_traits::receipt::gas_spent_by_transactions; diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index 670ee7f352ef..b64cf094042e 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -1365,14 +1365,14 @@ impl reth_codecs::Compact for TransactionSigned { let tx_bits = if zstd_bit { let mut tmp = Vec::with_capacity(256); if cfg!(feature = "std") { - crate::compression::TRANSACTION_COMPRESSOR.with(|compressor| { + reth_zstd_compressors::TRANSACTION_COMPRESSOR.with(|compressor| { let mut compressor = compressor.borrow_mut(); let tx_bits = self.transaction.to_compact(&mut tmp); buf.put_slice(&compressor.compress(&tmp).expect("Failed to compress")); tx_bits as u8 }) } else { - let mut compressor = crate::compression::create_tx_compressor(); + let mut compressor = reth_zstd_compressors::create_tx_compressor(); let tx_bits = self.transaction.to_compact(&mut tmp); buf.put_slice(&compressor.compress(&tmp).expect("Failed to compress")); tx_bits as u8 @@ -1399,7 +1399,7 @@ impl reth_codecs::Compact for TransactionSigned { let zstd_bit = bitflags >> 3; let (transaction, buf) = if zstd_bit != 0 { if cfg!(feature = "std") { - crate::compression::TRANSACTION_DECOMPRESSOR.with(|decompressor| { + reth_zstd_compressors::TRANSACTION_DECOMPRESSOR.with(|decompressor| { let mut decompressor = decompressor.borrow_mut(); // TODO: enforce that zstd is only present at a "top" level type @@ -1411,7 +1411,7 @@ impl reth_codecs::Compact for TransactionSigned { (transaction, buf) }) } else { - let mut decompressor = crate::compression::create_tx_decompressor(); + let mut decompressor = reth_zstd_compressors::create_tx_decompressor(); let transaction_type = (bitflags & 0b110) >> 1; let (transaction, _) = Transaction::from_compact(decompressor.decompress(buf), transaction_type); diff --git a/crates/storage/zstd-compressors/Cargo.toml b/crates/storage/zstd-compressors/Cargo.toml new file mode 100644 index 000000000000..357684f32fc1 --- /dev/null +++ b/crates/storage/zstd-compressors/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "reth-zstd-compressors" +version.workspace = true +edition.workspace = true +homepage.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +description = "Commonly used zstd compressors." + +[lints] +workspace = true + +[dependencies] +zstd = { workspace = true, features = ["experimental"] } + +[features] +default = ["std"] +std = [] \ No newline at end of file diff --git a/crates/primitives/src/compression/receipt_dictionary.bin b/crates/storage/zstd-compressors/receipt_dictionary.bin similarity index 100% rename from crates/primitives/src/compression/receipt_dictionary.bin rename to crates/storage/zstd-compressors/receipt_dictionary.bin diff --git a/crates/primitives/src/compression/mod.rs b/crates/storage/zstd-compressors/src/lib.rs similarity index 62% rename from crates/primitives/src/compression/mod.rs rename to crates/storage/zstd-compressors/src/lib.rs index ecceafc20682..d5167120bc76 100644 --- a/crates/primitives/src/compression/mod.rs +++ b/crates/storage/zstd-compressors/src/lib.rs @@ -1,41 +1,61 @@ +//! Commonly used zstd [`Compressor`] and [`Decompressor`] for reth types. + +#![doc( + html_logo_url = "https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-docs.png", + html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", + issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" +)] +#![cfg_attr(not(test), warn(unused_crate_dependencies))] +#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(not(feature = "std"), no_std)] + +extern crate alloc; + +use crate::alloc::string::ToString; use alloc::vec::Vec; -use core::cell::RefCell; use zstd::bulk::{Compressor, Decompressor}; /// Compression/Decompression dictionary for `Receipt`. -pub static RECEIPT_DICTIONARY: &[u8] = include_bytes!("./receipt_dictionary.bin"); +pub static RECEIPT_DICTIONARY: &[u8] = include_bytes!("../receipt_dictionary.bin"); /// Compression/Decompression dictionary for `Transaction`. -pub static TRANSACTION_DICTIONARY: &[u8] = include_bytes!("./transaction_dictionary.bin"); +pub static TRANSACTION_DICTIONARY: &[u8] = include_bytes!("../transaction_dictionary.bin"); -// We use `thread_local` compressors and decompressors because dictionaries can be quite big, and -// zstd-rs recommends to use one context/compressor per thread #[cfg(feature = "std")] -std::thread_local! { - /// Thread Transaction compressor. - pub static TRANSACTION_COMPRESSOR: RefCell> = RefCell::new( - Compressor::with_dictionary(0, TRANSACTION_DICTIONARY) - .expect("failed to initialize transaction compressor"), - ); - - /// Thread Transaction decompressor. - pub static TRANSACTION_DECOMPRESSOR: RefCell = - RefCell::new(ReusableDecompressor::new( - Decompressor::with_dictionary(TRANSACTION_DICTIONARY) - .expect("failed to initialize transaction decompressor"), - )); - - /// Thread receipt compressor. - pub static RECEIPT_COMPRESSOR: RefCell> = RefCell::new( - Compressor::with_dictionary(0, RECEIPT_DICTIONARY) - .expect("failed to initialize receipt compressor"), - ); - - /// Thread receipt decompressor. - pub static RECEIPT_DECOMPRESSOR: RefCell = - RefCell::new(ReusableDecompressor::new( - Decompressor::with_dictionary(RECEIPT_DICTIONARY) - .expect("failed to initialize receipt decompressor"), - )); +pub use locals::*; +#[cfg(feature = "std")] +mod locals { + use super::*; + use core::cell::RefCell; + + // We use `thread_local` compressors and decompressors because dictionaries can be quite big, + // and zstd-rs recommends to use one context/compressor per thread + std::thread_local! { + /// Thread Transaction compressor. + pub static TRANSACTION_COMPRESSOR: RefCell> = RefCell::new( + Compressor::with_dictionary(0, TRANSACTION_DICTIONARY) + .expect("failed to initialize transaction compressor"), + ); + + /// Thread Transaction decompressor. + pub static TRANSACTION_DECOMPRESSOR: RefCell = + RefCell::new(ReusableDecompressor::new( + Decompressor::with_dictionary(TRANSACTION_DICTIONARY) + .expect("failed to initialize transaction decompressor"), + )); + + /// Thread receipt compressor. + pub static RECEIPT_COMPRESSOR: RefCell> = RefCell::new( + Compressor::with_dictionary(0, RECEIPT_DICTIONARY) + .expect("failed to initialize receipt compressor"), + ); + + /// Thread receipt decompressor. + pub static RECEIPT_DECOMPRESSOR: RefCell = + RefCell::new(ReusableDecompressor::new( + Decompressor::with_dictionary(RECEIPT_DICTIONARY) + .expect("failed to initialize receipt decompressor"), + )); + } } /// Fn creates tx [`Compressor`] diff --git a/crates/primitives/src/compression/transaction_dictionary.bin b/crates/storage/zstd-compressors/transaction_dictionary.bin similarity index 100% rename from crates/primitives/src/compression/transaction_dictionary.bin rename to crates/storage/zstd-compressors/transaction_dictionary.bin