Track sent items using SmallVec

.github/workflows/bench.yml

@@ -27,6 +27,8 @@ jobs:
     defaults:
       run:
         shell: bash
+    env:
+      LD_LIBRARY_PATH: ${{ github.workspace }}/dist/Release/lib
 
     steps:
       - name: Checkout neqo

neqo-transport/src/ackrate.rs

@@ -11,8 +11,11 @@ use std::{cmp::max, time::Duration};
 use neqo_common::qtrace;
 
 use crate::{
-    connection::params::ACK_RATIO_SCALE, frame::FRAME_TYPE_ACK_FREQUENCY, packet::PacketBuilder,
-    recovery::RecoveryToken, stats::FrameStats,
+    connection::params::ACK_RATIO_SCALE,
+    frame::FRAME_TYPE_ACK_FREQUENCY,
+    packet::PacketBuilder,
+    recovery::{RecoveryToken, RecoveryTokenVec},
+    stats::FrameStats,
 };
 
 #[derive(Debug, Clone)]
@@ -105,7 +108,7 @@ impl FlexibleAckRate {
     fn write_frames(
         &mut self,
         builder: &mut PacketBuilder,
-        tokens: &mut Vec<RecoveryToken>,
+        tokens: &mut RecoveryTokenVec,
         stats: &mut FrameStats,
     ) {
         if !self.frame_outstanding
@@ -171,7 +174,7 @@ impl PeerAckDelay {
     pub fn write_frames(
         &mut self,
         builder: &mut PacketBuilder,
-        tokens: &mut Vec<RecoveryToken>,
+        tokens: &mut RecoveryTokenVec,
         stats: &mut FrameStats,
     ) {
         if let Self::Flexible(rate) = self {

neqo-transport/src/addr_valid.rs

@@ -19,7 +19,11 @@ use neqo_crypto::{
 use smallvec::SmallVec;
 
 use crate::{
-    cid::ConnectionId, packet::PacketBuilder, recovery::RecoveryToken, stats::FrameStats, Res,
+    cid::ConnectionId,
+    packet::PacketBuilder,
+    recovery::{RecoveryToken, RecoveryTokenVec},
+    stats::FrameStats,
+    Res,
 };
 
 /// A prefix we add to Retry tokens to distinguish them from `NEW_TOKEN` tokens.
@@ -346,7 +350,7 @@ impl NewTokenState {
     pub fn write_frames(
         &mut self,
         builder: &mut PacketBuilder,
-        tokens: &mut Vec<RecoveryToken>,
+        tokens: &mut RecoveryTokenVec,
         stats: &mut FrameStats,
     ) {
         if let Self::Server(ref mut sender) = self {
@@ -420,7 +424,7 @@ impl NewTokenSender {
     pub fn write_frames(
         &mut self,
         builder: &mut PacketBuilder,
-        tokens: &mut Vec<RecoveryToken>,
+        tokens: &mut RecoveryTokenVec,
         stats: &mut FrameStats,
     ) {
         for t in &mut self.tokens {

neqo-transport/src/cc/classic_cc.rs

@@ -621,7 +621,7 @@ mod tests {
             CongestionControl, CongestionControlAlgorithm, CWND_INITIAL_PKTS,
         },
         packet::{PacketNumber, PacketType},
-        recovery::SentPacket,
+        recovery::{RecoveryTokenVec, SentPacket},
         rtt::RttEstimate,
         Pmtud,
     };
@@ -656,7 +656,7 @@ mod tests {
             IpTosEcn::default(),
             now() + t,
             ack_eliciting,
-            Vec::new(),
+            RecoveryTokenVec::new(),
             100,
         )
     }
@@ -871,7 +871,7 @@ mod tests {
                     IpTosEcn::default(),
                     by_pto(t),
                     true,
-                    Vec::new(),
+                    RecoveryTokenVec::new(),
                     1000,
                 )
             })
@@ -993,7 +993,7 @@ mod tests {
             lost[0].ecn_mark(),
             lost[0].time_sent(),
             false,
-            Vec::new(),
+            RecoveryTokenVec::new(),
             lost[0].len(),
         );
         assert!(!persistent_congestion_by_pto(
@@ -1091,12 +1091,12 @@ mod tests {
             for _ in 0..packet_burst_size {
                 let p = SentPacket::new(
                     PacketType::Short,
-                    next_pn,
+                    next_pn, // pn
                     IpTosEcn::default(),
-                    now,
-                    true,
-                    Vec::new(),
-                    cc.max_datagram_size(),
+                    now,                     // time sent
+                    true,                    // ack eliciting
+                    RecoveryTokenVec::new(), // tokens
+                    cc.max_datagram_size(),  // size
                 );
                 next_pn += 1;
                 cc.on_packet_sent(&p, now);
@@ -1119,12 +1119,12 @@ mod tests {
         for _ in 0..ABOVE_APP_LIMIT_PKTS {
             let p = SentPacket::new(
                 PacketType::Short,
-                next_pn,
+                next_pn, // pn
                 IpTosEcn::default(),
-                now,
-                true,
-                Vec::new(),
-                cc.max_datagram_size(),
+                now,                     // time sent
+                true,                    // ack eliciting
+                RecoveryTokenVec::new(), // tokens
+                cc.max_datagram_size(),  // size
             );
             next_pn += 1;
             cc.on_packet_sent(&p, now);
@@ -1174,7 +1174,7 @@ mod tests {
             IpTosEcn::default(),
             now,
             true,
-            Vec::new(),
+            RecoveryTokenVec::new(),
             cc.max_datagram_size(),
         );
         cc.on_packet_sent(&p_lost, now);
@@ -1188,7 +1188,7 @@ mod tests {
             IpTosEcn::default(),
             now,
             true,
-            Vec::new(),
+            RecoveryTokenVec::new(),
             cc.max_datagram_size(),
         );
         cc.on_packet_sent(&p_not_lost, now);
@@ -1208,12 +1208,12 @@ mod tests {
             for _ in 0..packet_burst_size {
                 let p = SentPacket::new(
                     PacketType::Short,
-                    next_pn,
+                    next_pn, // pn
                     IpTosEcn::default(),
-                    now,
-                    true,
-                    Vec::new(),
-                    cc.max_datagram_size(),
+                    now,                     // time sent
+                    true,                    // ack eliciting
+                    RecoveryTokenVec::new(), // tokens
+                    cc.max_datagram_size(),  // size
                 );
                 next_pn += 1;
                 cc.on_packet_sent(&p, now);
@@ -1246,7 +1246,7 @@ mod tests {
                 IpTosEcn::default(),
                 now,
                 true,
-                Vec::new(),
+                RecoveryTokenVec::new(),
                 cc.max_datagram_size(),
             );
             next_pn += 1;
@@ -1286,7 +1286,7 @@ mod tests {
             IpTosEcn::default(),
             now,
             true,
-            Vec::new(),
+            RecoveryTokenVec::new(),
             cc.max_datagram_size(),
         );
         cc.on_packet_sent(&p_ce, now);

neqo-transport/src/cc/tests/cubic.rs

@@ -27,7 +27,7 @@ use crate::{
     },
     packet::PacketType,
     pmtud::Pmtud,
-    recovery::SentPacket,
+    recovery::{RecoveryTokenVec, SentPacket},
     rtt::RttEstimate,
 };
 
@@ -46,12 +46,12 @@ fn fill_cwnd(cc: &mut ClassicCongestionControl<Cubic>, mut next_pn: u64, now: In
     while cc.bytes_in_flight() < cc.cwnd() {
         let sent = SentPacket::new(
             PacketType::Short,
-            next_pn,
+            next_pn, // pn
             IpTosEcn::default(),
-            now,
-            true,
-            Vec::new(),
-            cc.max_datagram_size(),
+            now,                     // time sent
+            true,                    // ack eliciting
+            RecoveryTokenVec::new(), // tokens
+            cc.max_datagram_size(),  // size
         );
         cc.on_packet_sent(&sent, now);
         next_pn += 1;
@@ -66,7 +66,7 @@ fn ack_packet(cc: &mut ClassicCongestionControl<Cubic>, pn: u64, now: Instant) {
         IpTosEcn::default(),
         now,
         true,
-        Vec::new(),
+        RecoveryTokenVec::new(),
         cc.max_datagram_size(),
     );
     cc.on_packets_acked(&[acked], &RttEstimate::from_duration(RTT), now);
@@ -80,7 +80,7 @@ fn packet_lost(cc: &mut ClassicCongestionControl<Cubic>, pn: u64) {
         IpTosEcn::default(),
         now(),
         true,
-        Vec::new(),
+        RecoveryTokenVec::new(),
         cc.max_datagram_size(),
     );
     cc.on_packets_lost(None, None, PTO, &[p_lost], now());

neqo-transport/src/cc/tests/new_reno.rs

@@ -18,7 +18,7 @@ use crate::{
     cc::{new_reno::NewReno, ClassicCongestionControl, CongestionControl},
     packet::PacketType,
     pmtud::Pmtud,
-    recovery::SentPacket,
+    recovery::{RecoveryTokenVec, SentPacket},
     rtt::RttEstimate,
 };
 
@@ -52,7 +52,7 @@ fn issue_876() {
             IpTosEcn::default(),
             before,
             true,
-            Vec::new(),
+            RecoveryTokenVec::new(),
             cc.max_datagram_size() - 1,
         ),
         SentPacket::new(
@@ -61,7 +61,7 @@ fn issue_876() {
             IpTosEcn::default(),
             before,
             true,
-            Vec::new(),
+            RecoveryTokenVec::new(),
             cc.max_datagram_size() - 2,
         ),
         SentPacket::new(
@@ -70,7 +70,7 @@ fn issue_876() {
             IpTosEcn::default(),
             before,
             true,
-            Vec::new(),
+            RecoveryTokenVec::new(),
             cc.max_datagram_size(),
         ),
         SentPacket::new(
@@ -79,7 +79,7 @@ fn issue_876() {
             IpTosEcn::default(),
             before,
             true,
-            Vec::new(),
+            RecoveryTokenVec::new(),
             cc.max_datagram_size(),
         ),
         SentPacket::new(
@@ -88,7 +88,7 @@ fn issue_876() {
             IpTosEcn::default(),
             before,
             true,
-            Vec::new(),
+            RecoveryTokenVec::new(),
             cc.max_datagram_size(),
         ),
         SentPacket::new(
@@ -97,7 +97,7 @@ fn issue_876() {
             IpTosEcn::default(),
             before,
             true,
-            Vec::new(),
+            RecoveryTokenVec::new(),
             cc.max_datagram_size(),
         ),
         SentPacket::new(
@@ -106,7 +106,7 @@ fn issue_876() {
             IpTosEcn::default(),
             after,
             true,
-            Vec::new(),
+            RecoveryTokenVec::new(),
             cc.max_datagram_size() - 3,
         ),
     ];
@@ -162,7 +162,7 @@ fn issue_1465() {
             IpTosEcn::default(),
             now,
             true,
-            Vec::new(),
+            RecoveryTokenVec::new(),
             max_datagram_size,
         );
         pn += 1;

neqo-transport/src/cid.rs

@@ -19,8 +19,11 @@ use neqo_crypto::{random, randomize};
 use smallvec::{smallvec, SmallVec};
 
 use crate::{
-    frame::FRAME_TYPE_NEW_CONNECTION_ID, packet::PacketBuilder, recovery::RecoveryToken,
-    stats::FrameStats, Error, Res,
+    frame::FRAME_TYPE_NEW_CONNECTION_ID,
+    packet::PacketBuilder,
+    recovery::{RecoveryToken, RecoveryTokenVec},
+    stats::FrameStats,
+    Error, Res,
 };
 
 pub const MAX_CONNECTION_ID_LEN: usize = 20;
@@ -553,7 +556,7 @@ impl ConnectionIdManager {
     pub fn write_frames(
         &mut self,
         builder: &mut PacketBuilder,
-        tokens: &mut Vec<RecoveryToken>,
+        tokens: &mut RecoveryTokenVec,
         stats: &mut FrameStats,
     ) {
         if self.generator.deref().borrow().generates_empty_cids() {

neqo-transport/src/connection/idle.rs

@@ -11,7 +11,7 @@ use std::{
 
 use neqo_common::qtrace;
 
-use crate::recovery::RecoveryToken;
+use crate::recovery::{RecoveryToken, RecoveryTokenVec};
 
 #[derive(Debug, Clone)]
 /// There's a little bit of different behavior for resetting idle timeout. See
@@ -115,7 +115,7 @@ impl IdleTimeout {
         &mut self,
         now: Instant,
         pto: Duration,
-        tokens: &mut Vec<RecoveryToken>,
+        tokens: &mut RecoveryTokenVec,
     ) -> bool {
         if !self.keep_alive_outstanding && now >= self.keep_alive_timeout(now, pto) {
             self.keep_alive_outstanding = true;