Stopped store all content in memory

src/rpm/builder.rs

@@ -2,9 +2,7 @@ use std::collections::{BTreeMap, BTreeSet};
 use std::convert::TryInto;
 
 use std::fs;
-#[cfg(feature = "signature-meta")]
-use std::io;
-use std::io::{Read, Write};
+use std::io::{self, Read, Seek, Write};
 #[cfg(unix)]
 use std::os::unix::fs::PermissionsExt;
 
@@ -435,21 +433,23 @@ impl RPMBuilder {
     /// build without a signature
     ///
     /// ignores a present key, if any
-    pub fn build(self) -> Result<RPMPackage, RPMError> {
+    pub fn build(self) -> Result<RPMPackage<io::Cursor<Vec<u8>>>, RPMError> {
         let (lead, header_idx_tag, content) = self.prepare_data()?;
 
+        let mut content = io::Cursor::new(content);
         let mut header = Vec::with_capacity(128);
         header_idx_tag.write(&mut header)?;
 
         let digest_header = {
             let header = header;
-            let header_and_content_len = header.len() + content.len();
+            let header_and_content_len = header.len() as u64 + content.get_ref().len() as u64;
 
             let Digests {
                 header_and_content_digest: header_and_content_digest_md5,
                 header_digest_sha1,
                 header_digest_sha256,
-            } = RPMPackage::create_sig_header_digests(header.as_slice(), content.as_slice())?;
+            } = RPMPackage::create_sig_header_digests(header.as_slice(), &mut content)?;
+            content.rewind()?;
 
             Header::<IndexSignatureTag>::builder()
                 .add_digest(
@@ -473,23 +473,25 @@ impl RPMBuilder {
     ///
     /// See `signature::Signing` for more details.
     #[cfg(feature = "signature-meta")]
-    pub fn build_and_sign<S>(self, signer: S) -> Result<RPMPackage, RPMError>
+    pub fn build_and_sign<S>(self, signer: S) -> Result<RPMPackage<io::Cursor<Vec<u8>>>, RPMError>
     where
         S: signature::Signing<signature::algorithm::RSA>,
     {
         let (lead, header_idx_tag, content) = self.prepare_data()?;
 
+        let mut content = io::Cursor::new(content);
         let mut header = Vec::with_capacity(128);
         header_idx_tag.write(&mut header)?;
         let header = header;
 
-        let header_and_content_len = header.len() + content.len();
+        let header_and_content_len = header.len() as u64 + content.get_ref().len() as u64;
 
         let Digests {
             header_and_content_digest: header_and_content_digest_md5,
             header_digest_sha1,
             header_digest_sha256,
-        } = RPMPackage::create_sig_header_digests(header.as_slice(), content.as_slice())?;
+        } = RPMPackage::create_sig_header_digests(header.as_slice(), &mut content)?;
+        content.rewind()?;
 
         let builder = Header::<IndexSignatureTag>::builder().add_digest(
             header_digest_sha1.as_str(),
@@ -500,8 +502,9 @@ impl RPMBuilder {
         let signature_header = {
             let rsa_sig_header_only = signer.sign(header.as_slice())?;
 
-            let cursor = io::Cursor::new(header).chain(io::Cursor::new(&content));
+            let cursor = io::Cursor::new(header).chain(&mut content);
             let rsa_sig_header_and_archive = signer.sign(cursor)?;
+            content.rewind()?;
 
             builder
                 .add_signature(

src/rpm/headers/header.rs

@@ -329,7 +329,7 @@ impl Header<IndexSignatureTag> {
     /// Please use the [`builder`](Self::builder()) which has modular and safe API.
     #[cfg(feature = "signature-meta")]
     pub(crate) fn new_signature_header(
-        headers_plus_payload_size: usize,
+        headers_plus_payload_size: u64,
         md5sum: &[u8],
         sha1: &str,
         sha256: &str,
@@ -953,7 +953,7 @@ mod test {
         };
 
         let built = Header::<IndexSignatureTag>::new_signature_header(
-            size as usize,
+            size,
             md5sum,
             sha1,
             sha256,

src/rpm/headers/signature_builder.rs

@@ -56,13 +56,13 @@ where
     T: ConstructionStage,
 {
     /// Construct the complete signature header.
-    pub fn build(mut self, headers_plus_payload_size: usize) -> Header<IndexSignatureTag> {
+    pub fn build(mut self, headers_plus_payload_size: u64) -> Header<IndexSignatureTag> {
         self.entries.insert(
             0,
             IndexEntry::new(
                 IndexSignatureTag::RPMSIGTAG_LONGSIZE,
                 0i32, // externally filled
-                IndexData::Int64(vec![headers_plus_payload_size as u64]),
+                IndexData::Int64(vec![headers_plus_payload_size]),
             ),
         );
 

src/rpm/mod.rs

@@ -4,6 +4,7 @@ mod headers;
 mod package;
 
 pub mod signature;
+mod skip_reader;
 
 pub use headers::*;
 

src/rpm/package.rs

@@ -1,5 +1,6 @@
+use std::fmt;
 use std::fs;
-use std::io;
+use std::io::{self, BufRead, BufReader, Seek};
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
 
@@ -10,11 +11,12 @@ use num_traits::FromPrimitive;
 
 use crate::constants::*;
 use crate::errors::*;
+use crate::rpm::skip_reader::SkipReader;
 #[cfg(feature = "signature-meta")]
 use crate::signature;
 use crate::CompressionType;
 #[cfg(feature = "signature-meta")]
-use std::io::Read;
+use std::io::{Read, SeekFrom};
 
 use super::headers::*;
 use super::Lead;
@@ -38,53 +40,71 @@ pub struct Digests {
 ///
 /// Can either be created using the [`RPMPackageBuilder`](super::builder::RPMPackageBuilder)
 /// or used with [`parse`](`self::RPMPackage::parse`) to obtain from a file.
-#[derive(Debug)]
-pub struct RPMPackage {
+pub struct RPMPackage<R> {
     /// Header and metadata structures.
     ///
     /// Contains the constant lead as well as the metadata store.
     pub metadata: RPMPackageMetadata,
     /// The compressed or uncompressed files.
-    pub content: Vec<u8>,
+    pub content: R,
 }
 
-impl RPMPackage {
+#[cfg(feature = "signature-meta")]
+fn stream_len<T: Seek>(mut stream: T) -> io::Result<u64> {
+    let old_pos = stream.stream_position()?;
+    let len = stream.seek(SeekFrom::End(0))?;
+
+    // Avoid seeking a third time when we were already at the end of the
+    // stream. The branch is usually way cheaper than a seek operation.
+    if old_pos != len {
+        stream.seek(SeekFrom::Start(old_pos))?;
+    }
+
+    Ok(len)
+}
+
+impl RPMPackage<()> {
     /// Open and parse a file at the provided path as an RPM package
-    pub fn open<P: AsRef<Path>>(path: P) -> Result<Self, RPMError> {
+    pub fn open<P: AsRef<Path>>(
+        path: P,
+    ) -> Result<RPMPackage<SkipReader<BufReader<fs::File>>>, RPMError> {
         let rpm_file = fs::File::open(path.as_ref())?;
-        let mut buf_reader = io::BufReader::new(rpm_file);
-        Self::parse(&mut buf_reader)
+        let buf_reader = BufReader::new(rpm_file);
+        RPMPackage::parse(buf_reader)
     }
+}
 
+impl<R: BufRead + Seek> RPMPackage<R> {
     /// Parse an RPM package from an existing buffer
-    pub fn parse<T: io::BufRead>(input: &mut T) -> Result<Self, RPMError> {
-        let metadata = RPMPackageMetadata::parse(input)?;
-        let mut content = Vec::new();
-        input.read_to_end(&mut content)?;
+    pub fn parse(mut input: R) -> Result<RPMPackage<SkipReader<R>>, RPMError> {
+        let metadata = RPMPackageMetadata::parse(&mut input)?;
+        let content = SkipReader::new(input)?;
         Ok(RPMPackage { metadata, content })
     }
 
     /// Write the RPM package to a buffer
-    pub fn write<W: io::Write>(&self, out: &mut W) -> Result<(), RPMError> {
+    pub fn write<W: io::Write>(&mut self, out: &mut W) -> Result<(), RPMError> {
         self.metadata.write(out)?;
-        out.write_all(&self.content)?;
+        io::copy(&mut self.content, out)?;
+        self.content.rewind()?;
         Ok(())
     }
 
     /// Write the RPM package to a file
-    pub fn write_file<P: AsRef<Path>>(&self, path: P) -> Result<(), RPMError> {
+    pub fn write_file<P: AsRef<Path>>(&mut self, path: P) -> Result<(), RPMError> {
         self.write(&mut io::BufWriter::new(fs::File::create(path)?))
     }
 
     /// Prepare both header and content digests as used by the `SignatureIndex`.
     pub(crate) fn create_sig_header_digests(
         header: &[u8],
-        payload: &[u8],
+        payload: &mut R,
     ) -> Result<Digests, RPMError> {
         let digest_md5 = {
             let mut hasher = md5::Md5::default();
             hasher.update(header);
-            hasher.update(payload);
+            io::copy(payload, &mut hasher)?;
+            payload.rewind()?;
             let hash_result = hasher.finalize();
             hash_result.to_vec()
         };
@@ -112,20 +132,21 @@ impl RPMPackage {
         // make sure to not hash any previous signatures in the header
         self.metadata.header.write(&mut header_bytes)?;
 
-        let header_and_content_len = header_bytes.len() + self.content.len();
+        let content_len = stream_len(&mut self.content)?;
+        let header_and_content_len = header_bytes.len() as u64 + content_len;
 
         let Digests {
             header_digest_sha256,
             header_digest_sha1,
             header_and_content_digest,
-        } = Self::create_sig_header_digests(header_bytes.as_slice(), &self.content)?;
+        } = Self::create_sig_header_digests(header_bytes.as_slice(), &mut self.content)?;
 
         let rsa_signature_spanning_header_only = signer.sign(header_bytes.as_slice())?;
-        let mut header_and_content_cursor =
-            io::Cursor::new(header_bytes).chain(io::Cursor::new(&self.content));
+        let mut header_and_content_cursor = io::Cursor::new(header_bytes).chain(&mut self.content);
 
         let rsa_signature_spanning_header_and_archive =
             signer.sign(&mut header_and_content_cursor)?;
+        self.content.rewind()?;
 
         // NOTE: size stands for the combined size of header and payload.
         self.metadata.signature = Header::<IndexSignatureTag>::new_signature_header(
@@ -146,7 +167,7 @@ impl RPMPackage {
 
     /// Verify the signature as present within the RPM package.
     #[cfg(feature = "signature-meta")]
-    pub fn verify_signature<V>(&self, verifier: V) -> Result<(), RPMError>
+    pub fn verify_signature<V>(&mut self, verifier: V) -> Result<(), RPMError>
     where
         V: signature::Verifying<signature::algorithm::RSA, Signature = Vec<u8>>,
     {
@@ -159,6 +180,9 @@ impl RPMPackage {
             .get_entry_data_as_binary(IndexSignatureTag::RPMSIGTAG_RSA)?;
 
         signature::echo_signature("signature_header(header only)", signature_header_only);
+        verifier.verify(header_bytes.as_slice(), signature_header_only)?;
+
+        self.verify_digests()?;
 
         let signature_header_and_content = self
             .metadata
@@ -169,25 +193,21 @@ impl RPMPackage {
             "signature_header(header and content)",
             signature_header_and_content,
         );
-
-        verifier.verify(header_bytes.as_slice(), signature_header_only)?;
-        self.verify_digests()?;
-
-        let header_and_content_cursor =
-            io::Cursor::new(header_bytes).chain(io::Cursor::new(&self.content));
+        let header_and_content_cursor = io::Cursor::new(header_bytes).chain(&mut self.content);
         verifier.verify(header_and_content_cursor, signature_header_and_content)?;
+        self.content.rewind()?;
 
         Ok(())
     }
 
     /// Verify any digests which may be present in the RPM headers
-    pub fn verify_digests(&self) -> Result<(), RPMError> {
+    pub fn verify_digests(&mut self) -> Result<(), RPMError> {
         let mut header = Vec::<u8>::with_capacity(1024);
         // make sure to not hash any previous signatures in the header
         self.metadata.header.write(&mut header)?;
 
         let pkg_actual_digests =
-            Self::create_sig_header_digests(header.as_slice(), self.content.as_slice())?;
+            Self::create_sig_header_digests(header.as_slice(), &mut self.content)?;
 
         let md5 = self
             .metadata
@@ -246,7 +266,8 @@ impl RPMPackage {
                 // At the present moment even rpmbuild only supports sha256
             };
             let payload_digest = {
-                hasher.update(self.content.as_slice());
+                io::copy(&mut self.content, &mut hasher)?;
+                self.content.rewind()?;
                 hex::encode(hasher.finalize())
             };
             if payload_digest != payload_digest_val[0] {
@@ -258,6 +279,14 @@ impl RPMPackage {
     }
 }
 
+impl<R> fmt::Debug for RPMPackage<R> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("RPMPackage")
+            .field("metadata", &self.metadata)
+            .finish()
+    }
+}
+
 #[derive(PartialEq, Debug)]
 pub struct RPMPackageMetadata {
     pub lead: Lead,
@@ -269,12 +298,12 @@ impl RPMPackageMetadata {
     /// Open and parse RPMPackageMetadata from the file at the provided path
     pub fn open<P: AsRef<Path>>(path: P) -> Result<Self, RPMError> {
         let rpm_file = fs::File::open(path.as_ref())?;
-        let mut buf_reader = io::BufReader::new(rpm_file);
+        let mut buf_reader = BufReader::new(rpm_file);
         Self::parse(&mut buf_reader)
     }
 
     /// Parse RPMPackageMetadata from the provided reader
-    pub fn parse<T: io::BufRead>(input: &mut T) -> Result<Self, RPMError> {
+    pub fn parse<T: BufRead>(input: &mut T) -> Result<Self, RPMError> {
         let mut lead_buffer = [0; LEAD_SIZE as usize];
         input.read_exact(&mut lead_buffer)?;
         let lead = Lead::parse(&lead_buffer)?;

src/rpm/skip_reader.rs

@@ -0,0 +1,44 @@
+use std::io;
+use std::io::SeekFrom;
+
+pub struct SkipReader<R> {
+    reader: R,
+    start_pos: u64,
+}
+
+impl<R: io::Read + io::Seek> SkipReader<R> {
+    pub(crate) fn new(mut reader: R) -> io::Result<Self> {
+        let start_pos = reader.stream_position()?;
+        Ok(Self { reader, start_pos })
+    }
+}
+
+impl<R: io::Seek> io::Seek for SkipReader<R> {
+    fn seek(&mut self, pos: SeekFrom) -> io::Result<u64> {
+        match pos {
+            SeekFrom::Start(p) => self.reader.seek(SeekFrom::Start(p + self.start_pos)),
+            SeekFrom::Current(p) => self
+                .reader
+                .seek(SeekFrom::Current(p + self.start_pos as i64)),
+            SeekFrom::End(p) => self
+                .reader
+                .seek(SeekFrom::Current(p + self.start_pos as i64)),
+        }
+    }
+}
+
+impl<R: io::Read> io::Read for SkipReader<R> {
+    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
+        self.reader.read(buf)
+    }
+}
+
+impl<R: io::BufRead> io::BufRead for SkipReader<R> {
+    fn fill_buf(&mut self) -> io::Result<&[u8]> {
+        self.reader.fill_buf()
+    }
+
+    fn consume(&mut self, amt: usize) {
+        self.reader.consume(amt)
+    }
+}