Allow coding shred index to be different than data shred index

core/src/chacha.rs

@@ -166,7 +166,7 @@ mod tests {
         hasher.hash(&buf[..size]);
 
         //  golden needs to be updated if shred structure changes....
-        let golden: Hash = "9K6NR4cazo7Jzk2CpyXmNaZMGqvfXG83JzyJipkoHare"
+        let golden: Hash = "3z2WAkJp2dJjvpXsqsLbZ4muc39YGT7YY3eJQGtTHLfb"
             .parse()
             .unwrap();
 

core/src/window_service.rs

@@ -363,7 +363,7 @@ mod test {
         );
 
         // If it's a coding shred, test that slot >= root
-        let (common, coding) = Shredder::new_coding_shred_header(5, 5, 6, 6, 0, 0);
+        let (common, coding) = Shredder::new_coding_shred_header(5, 5, 5, 6, 6, 0, 0);
         let mut coding_shred =
             Shred::new_empty_from_header(common, DataShredHeader::default(), coding);
         Shredder::sign_shred(&leader_keypair, &mut coding_shred);

ledger/src/blocktree.rs

@@ -387,7 +387,11 @@ impl Blocktree {
                     "blocktree-erasure",
                     ("slot", slot as i64, i64),
                     ("start_index", set_index as i64, i64),
-                    ("end_index", erasure_meta.end_indexes().0 as i64, i64),
+                    (
+                        "end_index",
+                        (erasure_meta.set_index + erasure_meta.config.num_data() as u64) as i64,
+                        i64
+                    ),
                     ("recovery_attempted", attempted, bool),
                     ("recovery_status", status, String),
                     ("recovered", recovered as i64, i64),
@@ -420,8 +424,10 @@ impl Blocktree {
                             }
                         }
                     });
-                    (set_index..set_index + erasure_meta.config.num_coding() as u64).for_each(
-                        |i| {
+                    (erasure_meta.first_coding_index
+                        ..erasure_meta.first_coding_index
+                            + erasure_meta.config.num_coding() as u64)
+                        .for_each(|i| {
                             if let Some(shred) = prev_inserted_codes
                                 .remove(&(slot, i))
                                 .map(|s| {
@@ -450,13 +456,13 @@ impl Blocktree {
                             {
                                 available_shreds.push(shred);
                             }
-                        },
-                    );
+                        });
                     if let Ok(mut result) = Shredder::try_recovery(
                         available_shreds,
                         erasure_meta.config.num_data(),
                         erasure_meta.config.num_coding(),
                         set_index as usize,
+                        erasure_meta.first_coding_index as usize,
                         slot,
                     ) {
                         submit_metrics(true, "complete".into(), result.len());
@@ -683,17 +689,21 @@ impl Blocktree {
         if is_trusted
             || Blocktree::should_insert_coding_shred(&shred, index_meta.coding(), &self.last_root)
         {
-            let set_index = shred_index - u64::from(shred.coding_header.position);
+            let set_index = u64::from(shred.coding_header.fec_set_index);
             let erasure_config = ErasureConfig::new(
                 shred.coding_header.num_data_shreds as usize,
                 shred.coding_header.num_coding_shreds as usize,
             );
 
             let erasure_meta = erasure_metas.entry((slot, set_index)).or_insert_with(|| {
+                let first_coding_index =
+                    u64::from(shred.index()) - u64::from(shred.coding_header.position);
                 self.erasure_meta_cf
                     .get((slot, set_index))
                     .expect("Expect database get to succeed")
-                    .unwrap_or_else(|| ErasureMeta::new(set_index, &erasure_config))
+                    .unwrap_or_else(|| {
+                        ErasureMeta::new(set_index, first_coding_index, &erasure_config)
+                    })
             });
 
             if erasure_config != erasure_meta.config {
@@ -3716,7 +3726,8 @@ pub mod tests {
             let last_root = RwLock::new(0);
 
             let slot = 1;
-            let (mut shred, coding) = Shredder::new_coding_shred_header(slot, 11, 11, 11, 10, 0);
+            let (mut shred, coding) =
+                Shredder::new_coding_shred_header(slot, 11, 11, 11, 11, 10, 0);
             let coding_shred = Shred::new_empty_from_header(
                 shred.clone(),
                 DataShredHeader::default(),

ledger/src/blocktree_meta.rs

@@ -59,6 +59,8 @@ pub struct CodingIndex {
 pub struct ErasureMeta {
     /// Which erasure set in the slot this is
     pub set_index: u64,
+    /// First coding index in the FEC set
+    pub first_coding_index: u64,
     /// Size of shards in this erasure set
     pub size: usize,
     /// Erasure configuration for this erasure set
@@ -200,9 +202,10 @@ impl SlotMeta {
 }
 
 impl ErasureMeta {
-    pub fn new(set_index: u64, config: &ErasureConfig) -> ErasureMeta {
+    pub fn new(set_index: u64, first_coding_index: u64, config: &ErasureConfig) -> ErasureMeta {
         ErasureMeta {
             set_index,
+            first_coding_index,
             size: 0,
             config: *config,
         }
@@ -211,11 +214,12 @@ impl ErasureMeta {
     pub fn status(&self, index: &Index) -> ErasureMetaStatus {
         use ErasureMetaStatus::*;
 
-        let start_idx = self.start_index();
-        let (data_end_idx, coding_end_idx) = self.end_indexes();
-
-        let num_coding = index.coding().present_in_bounds(start_idx..coding_end_idx);
-        let num_data = index.data().present_in_bounds(start_idx..data_end_idx);
+        let num_coding = index.coding().present_in_bounds(
+            self.first_coding_index..self.first_coding_index + self.config.num_coding() as u64,
+        );
+        let num_data = index
+            .data()
+            .present_in_bounds(self.set_index..self.set_index + self.config.num_data() as u64);
 
         let (data_missing, coding_missing) = (
             self.config.num_data() - num_data,
@@ -240,23 +244,6 @@ impl ErasureMeta {
     pub fn size(&self) -> usize {
         self.size
     }
-
-    pub fn set_index_for(index: u64, num_data: usize) -> u64 {
-        index / num_data as u64
-    }
-
-    pub fn start_index(&self) -> u64 {
-        self.set_index
-    }
-
-    /// returns a tuple of (data_end, coding_end)
-    pub fn end_indexes(&self) -> (u64, u64) {
-        let start = self.start_index();
-        (
-            start + self.config.num_data() as u64,
-            start + self.config.num_coding() as u64,
-        )
-    }
 }
 
 #[cfg(test)]
@@ -272,7 +259,7 @@ mod test {
         let set_index = 0;
         let erasure_config = ErasureConfig::default();
 
-        let mut e_meta = ErasureMeta::new(set_index, &erasure_config);
+        let mut e_meta = ErasureMeta::new(set_index, set_index, &erasure_config);
         let mut rng = thread_rng();
         let mut index = Index::new(0);
         e_meta.size = 1;

ledger/src/shred.rs

@@ -28,7 +28,7 @@ use thiserror::Error;
 /// Constants are used over lazy_static for performance reasons.
 pub const SIZE_OF_COMMON_SHRED_HEADER: usize = 79;
 pub const SIZE_OF_DATA_SHRED_HEADER: usize = 3;
-pub const SIZE_OF_CODING_SHRED_HEADER: usize = 6;
+pub const SIZE_OF_CODING_SHRED_HEADER: usize = 10;
 pub const SIZE_OF_SIGNATURE: usize = 64;
 pub const SIZE_OF_DATA_SHRED_IGNORED_TAIL: usize =
     SIZE_OF_COMMON_SHRED_HEADER + SIZE_OF_CODING_SHRED_HEADER;
@@ -98,6 +98,7 @@ pub struct DataShredHeader {
 /// The coding shred header has FEC information
 #[derive(Serialize, Clone, Default, Deserialize, PartialEq, Debug)]
 pub struct CodingShredHeader {
+    pub fec_set_index: u32,
     pub num_data_shreds: u16,
     pub num_coding_shreds: u16,
     pub position: u16,
@@ -534,6 +535,7 @@ impl Shredder {
     pub fn new_coding_shred_header(
         slot: Slot,
         index: u32,
+        fec_set_index: u32,
         num_data: usize,
         num_code: usize,
         position: usize,
@@ -549,6 +551,7 @@ impl Shredder {
         (
             header,
             CodingShredHeader {
+                fec_set_index,
                 num_data_shreds: num_data as u16,
                 num_coding_shreds: num_code as u16,
                 position: position as u16,
@@ -585,6 +588,7 @@ impl Shredder {
                 let (header, coding_header) = Self::new_coding_shred_header(
                     slot,
                     start_index + i as u32,
+                    start_index,
                     num_data,
                     num_coding,
                     i,
@@ -615,6 +619,7 @@ impl Shredder {
                     let (common_header, coding_header) = Self::new_coding_shred_header(
                         slot,
                         start_index + i as u32,
+                        start_index,
                         num_data,
                         num_coding,
                         i,
@@ -672,6 +677,7 @@ impl Shredder {
         num_data: usize,
         num_coding: usize,
         first_index: usize,
+        first_code_index: usize,
         slot: Slot,
     ) -> std::result::Result<Vec<Shred>, reed_solomon_erasure::Error> {
         let mut recovered_data = vec![];
@@ -684,7 +690,8 @@ impl Shredder {
             let mut shred_bufs: Vec<Vec<u8>> = shreds
                 .into_iter()
                 .flat_map(|shred| {
-                    let index = Self::get_shred_index(&shred, num_data);
+                    let index =
+                        Self::get_shred_index(&shred, num_data, first_index, first_code_index);
                     let mut blocks = Self::fill_in_missing_shreds(
                         num_data,
                         num_coding,
@@ -782,11 +789,16 @@ impl Shredder {
         Ok(Self::reassemble_payload(num_data, data_shred_bufs))
     }
 
-    fn get_shred_index(shred: &Shred, num_data: usize) -> usize {
+    fn get_shred_index(
+        shred: &Shred,
+        num_data: usize,
+        first_data_index: usize,
+        first_code_index: usize,
+    ) -> usize {
         if shred.is_data() {
             shred.index() as usize
         } else {
-            shred.index() as usize + num_data
+            shred.index() as usize + num_data + first_data_index - first_code_index
         }
     }
 
@@ -1161,6 +1173,7 @@ pub mod tests {
                 num_data_shreds,
                 num_data_shreds,
                 0,
+                0,
                 slot
             ),
             Err(reed_solomon_erasure::Error::TooFewShardsPresent)
@@ -1172,6 +1185,7 @@ pub mod tests {
             num_data_shreds,
             num_data_shreds,
             0,
+            0,
             slot,
         )
         .unwrap();
@@ -1189,6 +1203,7 @@ pub mod tests {
             num_data_shreds,
             num_data_shreds,
             0,
+            0,
             slot,
         )
         .unwrap();
@@ -1236,6 +1251,7 @@ pub mod tests {
             num_data_shreds,
             num_data_shreds,
             0,
+            0,
             slot,
         )
         .unwrap();
@@ -1308,6 +1324,7 @@ pub mod tests {
             num_data_shreds,
             num_data_shreds,
             25,
+            25,
             slot,
         )
         .unwrap();
@@ -1339,6 +1356,7 @@ pub mod tests {
             num_data_shreds,
             num_data_shreds,
             25,
+            25,
             slot + 1,
         )
         .unwrap();
@@ -1351,14 +1369,15 @@ pub mod tests {
                 num_data_shreds,
                 num_data_shreds,
                 15,
+                15,
                 slot,
             ),
             Err(reed_solomon_erasure::Error::TooFewShardsPresent)
         );
 
         // Test8: Try recovery/reassembly with incorrect index. Hint: does not recover any shreds
         assert_matches!(
-            Shredder::try_recovery(shred_info, num_data_shreds, num_data_shreds, 35, slot,),
+            Shredder::try_recovery(shred_info, num_data_shreds, num_data_shreds, 35, 35, slot,),
             Err(reed_solomon_erasure::Error::TooFewShardsPresent)
         );
     }

ledger/tests/shred.rs

@@ -63,6 +63,7 @@ fn test_multi_fec_block_coding() {
             MAX_DATA_SHREDS_PER_FEC_BLOCK as usize,
             MAX_DATA_SHREDS_PER_FEC_BLOCK as usize,
             shred_start_index,
+            shred_start_index,
             slot,
         )
         .unwrap();