Add reference tick to data shreds (solana-labs#6772)

* Add reference tick to data shreds

* fix tests

core/benches/shredder.rs

@@ -35,7 +35,7 @@ fn bench_shredder_ticks(bencher: &mut Bencher) {
     let num_ticks = max_ticks_per_n_shreds(1) * num_shreds as u64;
     let entries = create_ticks(num_ticks, 0, Hash::default());
     bencher.iter(|| {
-        let shredder = Shredder::new(1, 0, RECOMMENDED_FEC_RATE, kp.clone()).unwrap();
+        let shredder = Shredder::new(1, 0, RECOMMENDED_FEC_RATE, kp.clone(), 0).unwrap();
         shredder.entries_to_shreds(&entries, true, 0);
     })
 }
@@ -50,7 +50,7 @@ fn bench_shredder_large_entries(bencher: &mut Bencher) {
     let entries = make_large_unchained_entries(txs_per_entry, num_entries);
     // 1Mb
     bencher.iter(|| {
-        let shredder = Shredder::new(1, 0, RECOMMENDED_FEC_RATE, kp.clone()).unwrap();
+        let shredder = Shredder::new(1, 0, RECOMMENDED_FEC_RATE, kp.clone(), 0).unwrap();
         shredder.entries_to_shreds(&entries, true, 0);
     })
 }
@@ -63,7 +63,7 @@ fn bench_deshredder(bencher: &mut Bencher) {
     let num_shreds = ((10000 * 1000) + (shred_size - 1)) / shred_size;
     let num_ticks = max_ticks_per_n_shreds(1) * num_shreds as u64;
     let entries = create_ticks(num_ticks, 0, Hash::default());
-    let shredder = Shredder::new(1, 0, RECOMMENDED_FEC_RATE, kp).unwrap();
+    let shredder = Shredder::new(1, 0, RECOMMENDED_FEC_RATE, kp, 0).unwrap();
     let data_shreds = shredder.entries_to_shreds(&entries, true, 0).0;
     bencher.iter(|| {
         let raw = &mut Shredder::deshred(&data_shreds).unwrap();
@@ -75,7 +75,7 @@ fn bench_deshredder(bencher: &mut Bencher) {
 fn bench_deserialize_hdr(bencher: &mut Bencher) {
     let data = vec![0; SIZE_OF_DATA_SHRED_PAYLOAD];
 
-    let shred = Shred::new_from_data(2, 1, 1, Some(&data), true, true);
+    let shred = Shred::new_from_data(2, 1, 1, Some(&data), true, true, 0);
 
     bencher.iter(|| {
         let payload = shred.payload.clone();

core/src/broadcast_stage/broadcast_fake_blobs_run.rs

@@ -44,6 +44,7 @@ impl BroadcastRun for BroadcastFakeBlobsRun {
             bank.parent().unwrap().slot(),
             RECOMMENDED_FEC_RATE,
             keypair.clone(),
+            (bank.tick_height() % bank.ticks_per_slot()) as u8,
         )
         .expect("Expected to create a new shredder");
 

core/src/broadcast_stage/fail_entry_verification_broadcast_run.rs

@@ -42,6 +42,7 @@ impl BroadcastRun for FailEntryVerificationBroadcastRun {
             bank.parent().unwrap().slot(),
             RECOMMENDED_FEC_RATE,
             keypair.clone(),
+            (bank.tick_height() % bank.ticks_per_slot()) as u8,
         )
         .expect("Expected to create a new shredder");
 

core/src/broadcast_stage/standard_broadcast_run.rs

@@ -2,7 +2,7 @@ use super::broadcast_utils::{self, ReceiveResults};
 use super::*;
 use crate::broadcast_stage::broadcast_utils::UnfinishedSlotInfo;
 use solana_ledger::entry::Entry;
-use solana_ledger::shred::{Shred, Shredder, RECOMMENDED_FEC_RATE};
+use solana_ledger::shred::{Shred, Shredder, RECOMMENDED_FEC_RATE, SHRED_TICK_REFERENCE_MASK};
 use solana_sdk::pubkey::Pubkey;
 use solana_sdk::signature::Keypair;
 use solana_sdk::timing::duration_as_us;
@@ -48,7 +48,7 @@ impl StandardBroadcastRun {
         }
     }
 
-    fn check_for_interrupted_slot(&mut self) -> Option<Shred> {
+    fn check_for_interrupted_slot(&mut self, max_ticks_in_slot: u8) -> Option<Shred> {
         let (slot, _) = self.current_slot_and_parent.unwrap();
         let mut last_unfinished_slot_shred = self
             .unfinished_slot
@@ -62,6 +62,7 @@ impl StandardBroadcastRun {
                         None,
                         true,
                         true,
+                        max_ticks_in_slot & SHRED_TICK_REFERENCE_MASK,
                     ))
                 } else {
                     None
@@ -83,13 +84,15 @@ impl StandardBroadcastRun {
         blocktree: &Blocktree,
         entries: &[Entry],
         is_slot_end: bool,
+        reference_tick: u8,
     ) -> (Vec<Shred>, Vec<Shred>) {
         let (slot, parent_slot) = self.current_slot_and_parent.unwrap();
         let shredder = Shredder::new(
             slot,
             parent_slot,
             RECOMMENDED_FEC_RATE,
             self.keypair.clone(),
+            reference_tick,
         )
         .expect("Expected to create a new shredder");
 
@@ -149,13 +152,15 @@ impl StandardBroadcastRun {
         let to_shreds_start = Instant::now();
 
         // 1) Check if slot was interrupted
-        let last_unfinished_slot_shred = self.check_for_interrupted_slot();
+        let last_unfinished_slot_shred =
+            self.check_for_interrupted_slot(bank.ticks_per_slot() as u8);
 
         // 2) Convert entries to shreds and coding shreds
         let (mut data_shreds, coding_shreds) = self.entries_to_shreds(
             blocktree,
             &receive_results.entries,
             last_tick_height == bank.max_tick_height(),
+            (bank.tick_height() % bank.ticks_per_slot()) as u8,
         );
         if let Some(last_shred) = last_unfinished_slot_shred {
             data_shreds.push(last_shred);
@@ -353,7 +358,7 @@ mod test {
 
         // Slot 2 interrupted slot 1
         let shred = run
-            .check_for_interrupted_slot()
+            .check_for_interrupted_slot(0)
             .expect("Expected a shred that signals an interrupt");
 
         // Validate the shred

core/src/chacha.rs

@@ -166,7 +166,7 @@ mod tests {
         hasher.hash(&buf[..size]);
 
         //  golden needs to be updated if blob stuff changes....
-        let golden: Hash = "BdmY3efqu7zbnFuGRAeFANwa35HkDdQ7hwhYez3xGXiM"
+        let golden: Hash = "HLzH7Nrh4q2K5WTh3e9vPNFZ1QVYhVDRMN9u5v51GqpJ"
             .parse()
             .unwrap();
 

core/src/replay_stage.rs

@@ -1137,7 +1137,7 @@ mod test {
             let payload_len = SIZE_OF_DATA_SHRED_PAYLOAD;
             let gibberish = [0xa5u8; PACKET_DATA_SIZE];
             let mut data_header = DataShredHeader::default();
-            data_header.flags = DATA_COMPLETE_SHRED;
+            data_header.flags |= DATA_COMPLETE_SHRED;
             let mut shred = Shred::new_empty_from_header(
                 ShredCommonHeader::default(),
                 data_header,

core/src/sigverify_shreds.rs

@@ -93,8 +93,15 @@ pub mod tests {
     #[test]
     fn test_sigverify_shreds_read_slots() {
         solana_logger::setup();
-        let mut shred =
-            Shred::new_from_data(0xdeadc0de, 0xc0de, 0xdead, Some(&[1, 2, 3, 4]), true, true);
+        let mut shred = Shred::new_from_data(
+            0xdeadc0de,
+            0xc0de,
+            0xdead,
+            Some(&[1, 2, 3, 4]),
+            true,
+            true,
+            0,
+        );
         let mut batch = [Packets::default(), Packets::default()];
 
         let keypair = Keypair::new();
@@ -103,8 +110,15 @@ pub mod tests {
         batch[0].packets[0].data[0..shred.payload.len()].copy_from_slice(&shred.payload);
         batch[0].packets[0].meta.size = shred.payload.len();
 
-        let mut shred =
-            Shred::new_from_data(0xc0dedead, 0xc0de, 0xdead, Some(&[1, 2, 3, 4]), true, true);
+        let mut shred = Shred::new_from_data(
+            0xc0dedead,
+            0xc0de,
+            0xdead,
+            Some(&[1, 2, 3, 4]),
+            true,
+            true,
+            0,
+        );
         Shredder::sign_shred(&keypair, &mut shred);
         batch[1].packets.resize(1, Packet::default());
         batch[1].packets[0].data[0..shred.payload.len()].copy_from_slice(&shred.payload);
@@ -127,12 +141,12 @@ pub mod tests {
         let mut batch = vec![Packets::default()];
         batch[0].packets.resize(2, Packet::default());
 
-        let mut shred = Shred::new_from_data(0, 0xc0de, 0xdead, Some(&[1, 2, 3, 4]), true, true);
+        let mut shred = Shred::new_from_data(0, 0xc0de, 0xdead, Some(&[1, 2, 3, 4]), true, true, 0);
         Shredder::sign_shred(&leader_keypair, &mut shred);
         batch[0].packets[0].data[0..shred.payload.len()].copy_from_slice(&shred.payload);
         batch[0].packets[0].meta.size = shred.payload.len();
 
-        let mut shred = Shred::new_from_data(0, 0xbeef, 0xc0de, Some(&[1, 2, 3, 4]), true, true);
+        let mut shred = Shred::new_from_data(0, 0xbeef, 0xc0de, Some(&[1, 2, 3, 4]), true, true, 0);
         let wrong_keypair = Keypair::new();
         Shredder::sign_shred(&wrong_keypair, &mut shred);
         batch[0].packets[1].data[0..shred.payload.len()].copy_from_slice(&shred.payload);

core/src/window_service.rs

@@ -315,7 +315,7 @@ mod test {
         parent: Slot,
         keypair: &Arc<Keypair>,
     ) -> Vec<Shred> {
-        let shredder = Shredder::new(slot, parent, 0.0, keypair.clone())
+        let shredder = Shredder::new(slot, parent, 0.0, keypair.clone(), 0)
             .expect("Failed to create entry shredder");
         shredder.entries_to_shreds(&entries, true, 0).0
     }

ledger/src/blocktree.rs

@@ -897,7 +897,7 @@ impl Blocktree {
         let mut remaining_ticks_in_slot = num_slots * ticks_per_slot - num_ticks_in_start_slot;
 
         let mut current_slot = start_slot;
-        let mut shredder = Shredder::new(current_slot, parent_slot, 0.0, keypair.clone())
+        let mut shredder = Shredder::new(current_slot, parent_slot, 0.0, keypair.clone(), 0)
             .expect("Failed to create entry shredder");
         let mut all_shreds = vec![];
         let mut slot_entries = vec![];
@@ -920,8 +920,14 @@ impl Blocktree {
                     shredder.entries_to_shreds(&current_entries, true, start_index);
                 all_shreds.append(&mut data_shreds);
                 all_shreds.append(&mut coding_shreds);
-                shredder = Shredder::new(current_slot, parent_slot, 0.0, keypair.clone())
-                    .expect("Failed to create entry shredder");
+                shredder = Shredder::new(
+                    current_slot,
+                    parent_slot,
+                    0.0,
+                    keypair.clone(),
+                    (ticks_per_slot - remaining_ticks_in_slot) as u8,
+                )
+                .expect("Failed to create entry shredder");
             }
 
             if entry.is_tick() {
@@ -1695,7 +1701,7 @@ pub fn create_new_ledger(ledger_path: &Path, genesis_block: &GenesisBlock) -> Re
     let entries = create_ticks(ticks_per_slot, hashes_per_tick, genesis_block.hash());
     let last_hash = entries.last().unwrap().hash;
 
-    let shredder = Shredder::new(0, 0, 0.0, Arc::new(Keypair::new()))
+    let shredder = Shredder::new(0, 0, 0.0, Arc::new(Keypair::new()), 0)
         .expect("Failed to create entry shredder");
     let shreds = shredder.entries_to_shreds(&entries, true, 0).0;
     assert!(shreds.last().unwrap().last_in_slot());
@@ -1780,7 +1786,7 @@ pub fn entries_to_test_shreds(
     parent_slot: Slot,
     is_full_slot: bool,
 ) -> Vec<Shred> {
-    let shredder = Shredder::new(slot, parent_slot, 0.0, Arc::new(Keypair::new()))
+    let shredder = Shredder::new(slot, parent_slot, 0.0, Arc::new(Keypair::new()), 0)
         .expect("Failed to create entry shredder");
 
     shredder.entries_to_shreds(&entries, is_full_slot, 0).0
@@ -3807,6 +3813,7 @@ pub mod tests {
                 Some(&[1, 1, 1]),
                 true,
                 true,
+                0,
             )];
 
             // With the corruption, nothing should be returned, even though an