From 8478f5c2d3643be033953c1dabd4f53a75295f4b Mon Sep 17 00:00:00 2001
From: yosoyubik <yosoyubik@gmail.com>
Date: Tue, 20 Aug 2024 10:21:11 +0200
Subject: [PATCH 01/22] mesa: account for null lss pairs

---
 pkg/vere/io/mesa.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/pkg/vere/io/mesa.c b/pkg/vere/io/mesa.c
index b90376ff16..ed07713947 100644
--- a/pkg/vere/io/mesa.c
+++ b/pkg/vere/io/mesa.c
@@ -2080,8 +2080,14 @@ _mesa_page_scry_jumbo_cb(void* vod_p, u3_noun res)
     lin_u->haz_y = lin_u->dat_y + dat_w;
     memcpy(lin_u->dat_y, dat_u->fra_y, dat_u->len_w);
 
+    c3_y* haz_y = lin_u->haz_y;
+    while ( pas != u3_nul ) {
+      u3r_bytes(0, 64, haz_y, u3h(pas));
+      haz_y += 64;
+      pas = u3t(pas);
+    }
+
     u3r_bytes(0, tip_w, lin_u->tip_y, pof);
-    u3r_bytes(0, haz_w, lin_u->haz_y, pas);
 
     mesa_free_pact(&jum_u);
   }

From 5b57c96009ffbe789cb7ac20e4f9d9ac415eafd2 Mon Sep 17 00:00:00 2001
From: yosoyubik <yosoyubik@gmail.com>
Date: Wed, 21 Aug 2024 16:06:58 +0200
Subject: [PATCH 02/22] mesa: correct 2-bit tag for 3-byte fragments

---
 pkg/vere/io/mesa/pact.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/vere/io/mesa/pact.c b/pkg/vere/io/mesa/pact.c
index f88fe72a51..075d91a119 100644
--- a/pkg/vere/io/mesa/pact.c
+++ b/pkg/vere/io/mesa/pact.c
@@ -294,7 +294,7 @@ _mesa_make_chub_tag(c3_d tot_d)
 {
   return  (tot_d <= 0xff)?       0b00 :
           (tot_d <= 0xffff)?     0b01 :
-          (tot_d <= 0xffffffff)? 0b10 :
+          (tot_d <= 0xffffff)?   0b10 :
                                  0b11;
 }
 

From 97ec30f76e757a295c60b8f2b71bb61351fbdd4d Mon Sep 17 00:00:00 2001
From: yosoyubik <yosoyubik@gmail.com>
Date: Wed, 21 Aug 2024 16:08:00 +0200
Subject: [PATCH 03/22] mesa: don't delete from pit after _mesa_req_pact_done

---
 pkg/vere/io/mesa.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/vere/io/mesa.c b/pkg/vere/io/mesa.c
index ed07713947..dc1017c003 100644
--- a/pkg/vere/io/mesa.c
+++ b/pkg/vere/io/mesa.c
@@ -2516,7 +2516,7 @@ _mesa_hear_page(u3_mesa_pict* pic_u, u3_lane lan_u)
   c3_y boq_y = 31;
   // c3_o done_with_jumbo_frame = __(0 == req_u->hav_d % boq_y);
   c3_o done_with_jumbo_frame = __(req_u->hav_d == req_u->tof_d); // TODO: fix for non-message-sized jumbo frames
-  _mesa_del_pit(sam_u, nam_u);
+  // _mesa_del_pit(sam_u, nam_u);  XX
   if ( c3y == done_with_jumbo_frame ) {
     u3_noun cad;
 

From 86f2840638f7f9361006fa27a9e3b1e6c638f12c Mon Sep 17 00:00:00 2001
From: yosoyubik <yosoyubik@gmail.com>
Date: Wed, 21 Aug 2024 16:09:09 +0200
Subject: [PATCH 04/22] mesa: early return after hearing fragment outside
 window

---
 pkg/vere/io/mesa.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/pkg/vere/io/mesa.c b/pkg/vere/io/mesa.c
index dc1017c003..a30ade7ee9 100644
--- a/pkg/vere/io/mesa.c
+++ b/pkg/vere/io/mesa.c
@@ -1234,8 +1234,12 @@ _mesa_req_pact_done(u3_pend_req*  req_u,
   if ( req_u->los_u->counter != nam_u->fra_d ) {
     if ( nam_u->fra_d < req_u->los_u->counter ) {
       u3l_log("fragment number too low: %"PRIu64, nam_u->fra_d);
+      c3_free(par_u);
+      return;
     } else if ( nam_u->fra_d >= req_u->los_u->counter + (sizeof(req_u->mis_u)/sizeof(u3_misord_buf)) ) {
       u3l_log("fragment number too high: %"PRIu64, nam_u->fra_d);
+      c3_free(par_u);
+      return;
     } else {
       // insert into misordered queue
       u3_misord_buf* buf_u = &req_u->mis_u[nam_u->fra_d - req_u->los_u->counter - 1];
@@ -1243,6 +1247,7 @@ _mesa_req_pact_done(u3_pend_req*  req_u,
       buf_u->len_w = dat_u->len_w;
       memcpy(buf_u->fra_y, dat_u->fra_y, dat_u->len_w);
       buf_u->par_u = par_u;
+      u3l_log("insert into misordered queue fra: %llu",  nam_u->fra_d );
     }
   }
   else if ( c3y != lss_verifier_ingest(req_u->los_u, dat_u->fra_y, dat_u->len_w, par_u) ) {

From 764ed6d4e7a1c15e35c545d1e9c5d9b1103d9379 Mon Sep 17 00:00:00 2001
From: yosoyubik <yosoyubik@gmail.com>
Date: Fri, 23 Aug 2024 16:28:47 +0200
Subject: [PATCH 05/22] mesa: only delete from bitset if fragmen was in flight

---
 pkg/vere/io/mesa.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkg/vere/io/mesa.c b/pkg/vere/io/mesa.c
index a30ade7ee9..6154825daf 100644
--- a/pkg/vere/io/mesa.c
+++ b/pkg/vere/io/mesa.c
@@ -1268,7 +1268,6 @@ _mesa_req_pact_done(u3_pend_req*  req_u,
     c3_free(par_u);
   }
 
-  bitset_del(&req_u->was_u, nam_u->fra_d);
   if ( nam_u->fra_d > req_u->ack_d ) {
     req_u->ack_d = nam_u->fra_d;
   }
@@ -1282,9 +1281,10 @@ _mesa_req_pact_done(u3_pend_req*  req_u,
   req_u->hav_d++;
 
   #ifdef MESA_DEBUG
-    u3l_log("fragment %"PRIu64" len %"PRIu64, nam_u->fra_d, req_u->hav_d);
+    u3l_log("fragment %llu hav_d %llu nex_d %llu ack_d %llu lef_d %llu old_d %llu", nam_u->fra_d, req_u->hav_d, req_u->nex_d, req_u->ack_d, req_u->lef_d, req_u->old_d);
   #endif
   if ( req_u->lef_d == nam_u->fra_d ) {
+    bitset_del(&req_u->was_u, nam_u->fra_d);
     req_u->lef_d++;
   }
 

From 1528badbc7ee17b893086e65ff76c980e702563c Mon Sep 17 00:00:00 2001
From: yosoyubik <yosoyubik@gmail.com>
Date: Fri, 23 Aug 2024 16:30:14 +0200
Subject: [PATCH 06/22] mesa: on acked page, resend up to the next fragment

---
 pkg/vere/io/mesa.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/vere/io/mesa.c b/pkg/vere/io/mesa.c
index 6154825daf..cfc3bcf83f 100644
--- a/pkg/vere/io/mesa.c
+++ b/pkg/vere/io/mesa.c
@@ -1305,7 +1305,7 @@ _mesa_req_pact_done(u3_pend_req*  req_u,
   c3_w siz_w = (1 << (nam_u->boq_y - 3));
   memcpy(req_u->dat_y + (siz_w * nam_u->fra_d), dat_u->fra_y, dat_u->len_w);
 
-  _try_resend(req_u, nam_u->fra_d);
+  _try_resend(req_u, req_u->nex_d);  // _try_resend(req_u, nam_u->fra_d);
   _update_resend_timer(req_u);
 }
 

From 1802bb649fef38ebcc16adbda48b1de01bef42c8 Mon Sep 17 00:00:00 2001
From: yosoyubik <yosoyubik@gmail.com>
Date: Tue, 27 Aug 2024 11:22:28 +0200
Subject: [PATCH 07/22] mesa: always bitset_del when adding to data buffer

---
 pkg/vere/io/mesa.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/vere/io/mesa.c b/pkg/vere/io/mesa.c
index cfc3bcf83f..4b0ed5ca23 100644
--- a/pkg/vere/io/mesa.c
+++ b/pkg/vere/io/mesa.c
@@ -1279,12 +1279,12 @@ _mesa_req_pact_done(u3_pend_req*  req_u,
   #endif
 
   req_u->hav_d++;
+  bitset_del(&req_u->was_u, nam_u->fra_d);
 
   #ifdef MESA_DEBUG
     u3l_log("fragment %llu hav_d %llu nex_d %llu ack_d %llu lef_d %llu old_d %llu", nam_u->fra_d, req_u->hav_d, req_u->nex_d, req_u->ack_d, req_u->lef_d, req_u->old_d);
   #endif
   if ( req_u->lef_d == nam_u->fra_d ) {
-    bitset_del(&req_u->was_u, nam_u->fra_d);
     req_u->lef_d++;
   }
 

From 95ef0b13972bf0406ea8cd17adc0d850d8c02af8 Mon Sep 17 00:00:00 2001
From: yosoyubik <yosoyubik@gmail.com>
Date: Tue, 27 Aug 2024 11:23:30 +0200
Subject: [PATCH 08/22] mesa: add out-of-window fragments to data buffer

---
 pkg/vere/io/mesa.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/pkg/vere/io/mesa.c b/pkg/vere/io/mesa.c
index 4b0ed5ca23..6a2456b2f3 100644
--- a/pkg/vere/io/mesa.c
+++ b/pkg/vere/io/mesa.c
@@ -1234,12 +1234,12 @@ _mesa_req_pact_done(u3_pend_req*  req_u,
   if ( req_u->los_u->counter != nam_u->fra_d ) {
     if ( nam_u->fra_d < req_u->los_u->counter ) {
       u3l_log("fragment number too low: %"PRIu64, nam_u->fra_d);
-      c3_free(par_u);
-      return;
+      // c3_free(par_u);
+      // return;
     } else if ( nam_u->fra_d >= req_u->los_u->counter + (sizeof(req_u->mis_u)/sizeof(u3_misord_buf)) ) {
       u3l_log("fragment number too high: %"PRIu64, nam_u->fra_d);
-      c3_free(par_u);
-      return;
+      // c3_free(par_u);
+      // return;
     } else {
       // insert into misordered queue
       u3_misord_buf* buf_u = &req_u->mis_u[nam_u->fra_d - req_u->los_u->counter - 1];

From 1103e9bea7400082c5d13a82a822eee3ee7e2306 Mon Sep 17 00:00:00 2001
From: yosoyubik <yosoyubik@gmail.com>
Date: Tue, 27 Aug 2024 16:36:12 +0200
Subject: [PATCH 09/22] mesa: fix misorder queue update

we also move forward from lef_d to account for the misordered that we have received

(note: check that this is right)
---
 pkg/vere/io/mesa.c | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/pkg/vere/io/mesa.c b/pkg/vere/io/mesa.c
index 6a2456b2f3..ea19c6d5c5 100644
--- a/pkg/vere/io/mesa.c
+++ b/pkg/vere/io/mesa.c
@@ -1188,9 +1188,15 @@ _mesa_burn_misorder_queue(u3_pend_req* req_u)
     }
   }
   // ratchet forward
-  num_w++; // account for the packet processed in _mesa_req_pact_done
-  memcpy(req_u->mis_u, req_u->mis_u + num_w, max_w - num_w);
-  memset(req_u->mis_u + max_w - num_w, 0, num_w * sizeof(u3_misord_buf));
+  u3l_log("updating lef_d %u", req_u->lef_d);
+  req_u->lef_d += num_w;  //  XX could we be skipping some lef_d's that we have not received?
+  u3l_log("updated lef_d %u num_w: %u", req_u->lef_d, num_w);
+
+  memset(req_u->mis_u, 0, num_w * sizeof(u3_misord_buf));
+  memcpy(req_u->mis_u,
+         (c3_y*)req_u->mis_u + (num_w * sizeof(u3_misord_buf)),
+         (max_w - num_w) * sizeof(u3_misord_buf));
+
   return res_o;
 }
 

From f9a18a8d45ba81e3b35a4356bca565ea2439093a Mon Sep 17 00:00:00 2001
From: yosoyubik <yosoyubik@gmail.com>
Date: Tue, 27 Aug 2024 16:36:48 +0200
Subject: [PATCH 10/22] mesa: restore early return for out of window fragments

---
 pkg/vere/io/mesa.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/pkg/vere/io/mesa.c b/pkg/vere/io/mesa.c
index ea19c6d5c5..ce2138e7cf 100644
--- a/pkg/vere/io/mesa.c
+++ b/pkg/vere/io/mesa.c
@@ -1240,12 +1240,12 @@ _mesa_req_pact_done(u3_pend_req*  req_u,
   if ( req_u->los_u->counter != nam_u->fra_d ) {
     if ( nam_u->fra_d < req_u->los_u->counter ) {
       u3l_log("fragment number too low: %"PRIu64, nam_u->fra_d);
-      // c3_free(par_u);
-      // return;
+      c3_free(par_u);
+      return;
     } else if ( nam_u->fra_d >= req_u->los_u->counter + (sizeof(req_u->mis_u)/sizeof(u3_misord_buf)) ) {
       u3l_log("fragment number too high: %"PRIu64, nam_u->fra_d);
-      // c3_free(par_u);
-      // return;
+      c3_free(par_u);
+      return;
     } else {
       // insert into misordered queue
       u3_misord_buf* buf_u = &req_u->mis_u[nam_u->fra_d - req_u->los_u->counter - 1];

From 2be8c02a9621b0c7f97368d4fe66efb3eba17855 Mon Sep 17 00:00:00 2001
From: yosoyubik <yosoyubik@gmail.com>
Date: Tue, 27 Aug 2024 16:40:05 +0200
Subject: [PATCH 11/22] mesa: bitset_del for misordered and lef_d fragment

---
 pkg/vere/io/mesa.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pkg/vere/io/mesa.c b/pkg/vere/io/mesa.c
index ce2138e7cf..8adf5da729 100644
--- a/pkg/vere/io/mesa.c
+++ b/pkg/vere/io/mesa.c
@@ -1254,6 +1254,7 @@ _mesa_req_pact_done(u3_pend_req*  req_u,
       memcpy(buf_u->fra_y, dat_u->fra_y, dat_u->len_w);
       buf_u->par_u = par_u;
       u3l_log("insert into misordered queue fra: %llu",  nam_u->fra_d );
+      bitset_del(&req_u->was_u, nam_u->fra_d);
     }
   }
   else if ( c3y != lss_verifier_ingest(req_u->los_u, dat_u->fra_y, dat_u->len_w, par_u) ) {
@@ -1285,12 +1286,12 @@ _mesa_req_pact_done(u3_pend_req*  req_u,
   #endif
 
   req_u->hav_d++;
-  bitset_del(&req_u->was_u, nam_u->fra_d);
 
   #ifdef MESA_DEBUG
     u3l_log("fragment %llu hav_d %llu nex_d %llu ack_d %llu lef_d %llu old_d %llu", nam_u->fra_d, req_u->hav_d, req_u->nex_d, req_u->ack_d, req_u->lef_d, req_u->old_d);
   #endif
   if ( req_u->lef_d == nam_u->fra_d ) {
+    bitset_del(&req_u->was_u, nam_u->fra_d);
     req_u->lef_d++;
   }
 

From c4dca5feb9c4e2e258ecbcf696e0cc0be3b2d051 Mon Sep 17 00:00:00 2001
From: yosoyubik <yosoyubik@gmail.com>
Date: Tue, 27 Aug 2024 16:41:00 +0200
Subject: [PATCH 12/22] mesa: fix memory leak

---
 pkg/vere/io/mesa.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/pkg/vere/io/mesa.c b/pkg/vere/io/mesa.c
index 8adf5da729..5e8c20ca1d 100644
--- a/pkg/vere/io/mesa.c
+++ b/pkg/vere/io/mesa.c
@@ -2633,6 +2633,8 @@ _mesa_hear_peek(u3_mesa_pict* pic_u, u3_lane lan_u)
   // otherwise, scry
   lin_u = c3_calloc(sizeof(u3_mesa_line));
   lin_u->typ_y = CTAG_WAIT;
+  _mesa_copy_name(&lin_u->nam_u, &pac_u->pek_u.nam_u);  // XX
+
   _mesa_put_jumbo_cache(sam_u, &pac_u->pek_u.nam_u, lin_u);
   u3_noun sky = _name_to_jumbo_scry(&pac_u->pek_u.nam_u);
   u3_noun our = u3i_chubs(2, sam_u->car_u.pir_u->who_d);

From c60fbb25deb2cf7e4adf95c04e67a47681e00058 Mon Sep 17 00:00:00 2001
From: yosoyubik <yosoyubik@gmail.com>
Date: Tue, 27 Aug 2024 16:41:38 +0200
Subject: [PATCH 13/22] mesa WIP fix resend timer

---
 pkg/vere/io/mesa.c | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/pkg/vere/io/mesa.c b/pkg/vere/io/mesa.c
index 5e8c20ca1d..701cc485a5 100644
--- a/pkg/vere/io/mesa.c
+++ b/pkg/vere/io/mesa.c
@@ -1155,10 +1155,18 @@ _update_resend_timer(u3_pend_req *req_u)
   }
   req_u->old_d = idx_d;
   req_u->tim_u.data = req_u;
-  c3_d gap_d = req_u->wat_u[idx_d].sen_d - now_d;
-  /* u3l_log("timeout %llu", (gag_u->rto_w - gap_d) / 1000); */
-  c3_w dur_w = (req_u->gag_u->rto_w - gap_d) / 1000;
-  uv_timer_start(&req_u->tim_u, _mesa_packet_timeout, dur_w, 0);
+  c3_d gap_d = req_u->wat_u[idx_d].sen_d == 0 ?
+                0 :
+                now_d - req_u->wat_u[idx_d].sen_d;
+  // if ( gap_d > req_u->gag_u->rto_w ) {
+  //   u3l_log("OMG %llu", gap_d);
+  // }
+  // u3l_log("timeout %llu", (req_u->gag_u->rto_w - gap_d) / 1000);
+  // c3_w dur_w = (req_u->gag_u->rto_w - gap_d) / 1000;
+
+  c3_d next_expiry = (gap_d + req_u->gag_u->rto_w) / 1000;
+    // now_d == gap_d ? req_u->gag_u->rto_w  : req_u->gag_u->rto_w - gap_d / 1000;
+  uv_timer_start(&req_u->tim_u, _mesa_packet_timeout, next_expiry, 0);
 }
 
 /* _mesa_packet_timeout(): callback for packet timeout

From 05fe7e9e0a2099d8292cf915e2efd7c1d513ab2c Mon Sep 17 00:00:00 2001
From: yosoyubik <yosoyubik@gmail.com>
Date: Wed, 28 Aug 2024 15:09:12 +0200
Subject: [PATCH 14/22] mesa: fix remaining fragments

---
 pkg/vere/io/mesa.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/vere/io/mesa.c b/pkg/vere/io/mesa.c
index 701cc485a5..0ae7b7d155 100644
--- a/pkg/vere/io/mesa.c
+++ b/pkg/vere/io/mesa.c
@@ -802,7 +802,7 @@ static void _mesa_handle_ack(u3_gage* gag_u, u3_pact_stat* pat_u)
 static inline c3_d
 _mesa_req_get_remaining(u3_pend_req* req_u)
 {
-  return req_u->tof_d - req_u->nex_d;
+  return req_u->tof_d - req_u->hav_d;
 }
 
 /*

From b756796f5c2f67502eeef3373906ae9405f851a0 Mon Sep 17 00:00:00 2001
From: yosoyubik <yosoyubik@gmail.com>
Date: Wed, 28 Aug 2024 15:09:47 +0200
Subject: [PATCH 15/22] mesa: init congestion to microseconds

---
 pkg/vere/io/mesa.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/pkg/vere/io/mesa.c b/pkg/vere/io/mesa.c
index 0ae7b7d155..2289429ce9 100644
--- a/pkg/vere/io/mesa.c
+++ b/pkg/vere/io/mesa.c
@@ -505,11 +505,11 @@ _mesa_request_key(u3_mesa_name* nam_u)
 }
 
 static void
-_init_gage(u3_gage* gag_u)
+_init_gage(u3_gage* gag_u)  //  microseconds
 {
-  gag_u->rto_w = 1000000;
-  gag_u->rtt_w = 1000000;
-  gag_u->rtv_w = 1000000;
+  gag_u->rto_w = 1000 * 1000 * 1000;  // ~s1
+  gag_u->rtt_w = 1000 * 1000 * 1000;  // ~s1
+  gag_u->rtv_w = 1000 * 1000 * 1000;  // ~s1
   gag_u->con_w = 0;
   gag_u->wnd_w = 1;
   gag_u->sst_w = 10000;

From 2895f5d23d9e8c76e8b8c00160345a1b3547ab0f Mon Sep 17 00:00:00 2001
From: yosoyubik <yosoyubik@gmail.com>
Date: Wed, 28 Aug 2024 15:10:16 +0200
Subject: [PATCH 16/22] mesa: update resent packet send time

---
 pkg/vere/io/mesa.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/vere/io/mesa.c b/pkg/vere/io/mesa.c
index 2289429ce9..22d65905db 100644
--- a/pkg/vere/io/mesa.c
+++ b/pkg/vere/io/mesa.c
@@ -1109,7 +1109,7 @@ _try_resend(u3_pend_req* req_u, c3_d ack_d)
     if ( (c3y == bitset_has(&req_u->was_u, i_d)) &&
         (now_d - req_u->wat_u[i_d].sen_d > req_u->gag_u->rto_w) ) {
       los_o = c3y;
-
+      _mesa_req_pact_resent(req_u, &pac_u->pek_u.nam_u);
       pac_u->pek_u.nam_u.fra_d = i_d;
       c3_w len_w = mesa_etch_pact_to_buf(buf_y, PACT_SIZE, pac_u);
       _mesa_send_modal(req_u->per_u, buf_y, len_w);

From f6797424b3b7afc7bad5b44215ee0d07f3f3f08a Mon Sep 17 00:00:00 2001
From: yosoyubik <yosoyubik@gmail.com>
Date: Wed, 28 Aug 2024 15:11:34 +0200
Subject: [PATCH 17/22] mesa: use RTO as next timer resend

---
 pkg/vere/io/mesa.c | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

diff --git a/pkg/vere/io/mesa.c b/pkg/vere/io/mesa.c
index 22d65905db..7680c6a9a9 100644
--- a/pkg/vere/io/mesa.c
+++ b/pkg/vere/io/mesa.c
@@ -1155,18 +1155,12 @@ _update_resend_timer(u3_pend_req *req_u)
   }
   req_u->old_d = idx_d;
   req_u->tim_u.data = req_u;
-  c3_d gap_d = req_u->wat_u[idx_d].sen_d == 0 ?
-                0 :
-                now_d - req_u->wat_u[idx_d].sen_d;
-  // if ( gap_d > req_u->gag_u->rto_w ) {
-  //   u3l_log("OMG %llu", gap_d);
-  // }
-  // u3l_log("timeout %llu", (req_u->gag_u->rto_w - gap_d) / 1000);
-  // c3_w dur_w = (req_u->gag_u->rto_w - gap_d) / 1000;
-
-  c3_d next_expiry = (gap_d + req_u->gag_u->rto_w) / 1000;
-    // now_d == gap_d ? req_u->gag_u->rto_w  : req_u->gag_u->rto_w - gap_d / 1000;
-  uv_timer_start(&req_u->tim_u, _mesa_packet_timeout, next_expiry, 0);
+  // c3_d gap_d = req_u->wat_u[idx_d].sen_d == 0 ?
+  //               0 :
+  //               now_d - req_u->wat_u[idx_d].sen_d;
+  c3_d next_expiry = req_u->gag_u->rto_w;
+  u3l_log("next_expiry %llu", next_expiry / 1000);
+  uv_timer_start(&req_u->tim_u, _mesa_packet_timeout, next_expiry / 1000, 0);
 }
 
 /* _mesa_packet_timeout(): callback for packet timeout

From 9c4a01db75688fc33f1999a6d1b78a891a1f676e Mon Sep 17 00:00:00 2001
From: yosoyubik <yosoyubik@gmail.com>
Date: Wed, 28 Aug 2024 15:12:31 +0200
Subject: [PATCH 18/22] mesa: burn misorder queue on out-ot-window fragments

---
 pkg/vere/io/mesa.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pkg/vere/io/mesa.c b/pkg/vere/io/mesa.c
index 7680c6a9a9..f25efaadae 100644
--- a/pkg/vere/io/mesa.c
+++ b/pkg/vere/io/mesa.c
@@ -1246,6 +1246,7 @@ _mesa_req_pact_done(u3_pend_req*  req_u,
       return;
     } else if ( nam_u->fra_d >= req_u->los_u->counter + (sizeof(req_u->mis_u)/sizeof(u3_misord_buf)) ) {
       u3l_log("fragment number too high: %"PRIu64, nam_u->fra_d);
+      _mesa_burn_misorder_queue(req_u);
       c3_free(par_u);
       return;
     } else {

From 5c932e9de47094140fc133bb1108e7b71948da01 Mon Sep 17 00:00:00 2001
From: yosoyubik <yosoyubik@gmail.com>
Date: Wed, 28 Aug 2024 15:13:35 +0200
Subject: [PATCH 19/22] mesa: don't try to resend after ack

---
 pkg/vere/io/mesa.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pkg/vere/io/mesa.c b/pkg/vere/io/mesa.c
index f25efaadae..b14735fe2e 100644
--- a/pkg/vere/io/mesa.c
+++ b/pkg/vere/io/mesa.c
@@ -1315,7 +1315,8 @@ _mesa_req_pact_done(u3_pend_req*  req_u,
   c3_w siz_w = (1 << (nam_u->boq_y - 3));
   memcpy(req_u->dat_y + (siz_w * nam_u->fra_d), dat_u->fra_y, dat_u->len_w);
 
-  _try_resend(req_u, req_u->nex_d);  // _try_resend(req_u, nam_u->fra_d);
+  // _try_resend(req_u, req_u->nex_d);
+  // _try_resend(req_u, nam_u->fra_d);
   _update_resend_timer(req_u);
 }
 

From a7821f28fb7de997533132fc6e0c001bfbab38cb Mon Sep 17 00:00:00 2001
From: yosoyubik <yosoyubik@gmail.com>
Date: Wed, 28 Aug 2024 15:14:14 +0200
Subject: [PATCH 20/22] mesa: use correct length when inserting jumbo frame

---
 pkg/vere/io/mesa.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/vere/io/mesa.c b/pkg/vere/io/mesa.c
index b14735fe2e..f74554dea5 100644
--- a/pkg/vere/io/mesa.c
+++ b/pkg/vere/io/mesa.c
@@ -2548,7 +2548,7 @@ _mesa_hear_page(u3_mesa_pict* pic_u, u3_lane lan_u)
         pac_u->pag_u.nam_u.boq_y = boq_y;
         pac_u->pag_u.dat_u.tob_d = req_u->tob_d;
         pac_u->pag_u.nam_u.fra_d = (req_u->hav_d >> boq_y);
-        pac_u->pag_u.dat_u.len_w += (1024 * (lev_d - 1));
+        pac_u->pag_u.dat_u.len_w = req_u->tob_d;
         pac_u->pag_u.dat_u.fra_y = req_u->dat_y;
         pac_u->pag_u.dat_u.aut_u = req_u->aut_u;
 

From f19e314638eb12a94023bc184ca2fcdc1d038a63 Mon Sep 17 00:00:00 2001
From: yosoyubik <yosoyubik@gmail.com>
Date: Wed, 28 Aug 2024 15:31:57 +0200
Subject: [PATCH 21/22] mesa: remove misordered queue handling

---
 pkg/vere/io/mesa.c | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/pkg/vere/io/mesa.c b/pkg/vere/io/mesa.c
index f74554dea5..182be7b8d8 100644
--- a/pkg/vere/io/mesa.c
+++ b/pkg/vere/io/mesa.c
@@ -1249,16 +1249,17 @@ _mesa_req_pact_done(u3_pend_req*  req_u,
       _mesa_burn_misorder_queue(req_u);
       c3_free(par_u);
       return;
-    } else {
-      // insert into misordered queue
-      u3_misord_buf* buf_u = &req_u->mis_u[nam_u->fra_d - req_u->los_u->counter - 1];
-      buf_u->fra_y = c3_calloc(dat_u->len_w);
-      buf_u->len_w = dat_u->len_w;
-      memcpy(buf_u->fra_y, dat_u->fra_y, dat_u->len_w);
-      buf_u->par_u = par_u;
-      u3l_log("insert into misordered queue fra: %llu",  nam_u->fra_d );
-      bitset_del(&req_u->was_u, nam_u->fra_d);
-    }
+    } //else { // XXXX
+    //   // insert into misordered queue
+    //                                       //  (4 - 3) - 1
+    //   u3_misord_buf* buf_u = &req_u->mis_u[nam_u->fra_d - req_u->los_u->counter - 1];
+    //   buf_u->fra_y = c3_calloc(dat_u->len_w);
+    //   buf_u->len_w = dat_u->len_w;
+    //   memcpy(buf_u->fra_y, dat_u->fra_y, dat_u->len_w);
+    //   buf_u->par_u = par_u;
+    //   u3l_log("insert into misordered queue fra: %llu",  nam_u->fra_d );
+    //   bitset_del(&req_u->was_u, nam_u->fra_d);
+    // }
   }
   else if ( c3y != lss_verifier_ingest(req_u->los_u, dat_u->fra_y, dat_u->len_w, par_u) ) {
     u3l_log("auth fail frag %"PRIu64, nam_u->fra_d);

From 6689197b593d3751704de2b2afeef2c42bd8fa28 Mon Sep 17 00:00:00 2001
From: yosoyubik <yosoyubik@gmail.com>
Date: Thu, 29 Aug 2024 19:40:07 +0200
Subject: [PATCH 22/22] mesa: fix misordered queue handling

---
 pkg/vere/io/mesa.c | 80 ++++++++++++++++++++--------------------------
 1 file changed, 35 insertions(+), 45 deletions(-)

diff --git a/pkg/vere/io/mesa.c b/pkg/vere/io/mesa.c
index 182be7b8d8..56caa1f6a8 100644
--- a/pkg/vere/io/mesa.c
+++ b/pkg/vere/io/mesa.c
@@ -1046,7 +1046,7 @@ _mesa_send_modal(u3_peer* per_u, c3_y* buf_y, c3_w len_w)
   if ( ( c3y == _mesa_is_direct_mode(per_u) ) ||
        // if we are the sponsor of the ship, don't send to ourselves
        (our_o == c3y) )  {
-    u3l_log("mesa: direct");
+    // u3l_log("mesa: direct");
     _mesa_send_buf(sam_u, per_u->dan_u, sen_y, len_w);
     per_u->dir_u.sen_d = now_d;
   }
@@ -1117,7 +1117,7 @@ _try_resend(u3_pend_req* req_u, c3_d ack_d)
   }
 
   if ( c3y == los_o ) {
-    req_u->gag_u->sst_w = (req_u->gag_u->wnd_w / 2) + 1;
+    req_u->gag_u->sst_w = c3_max(1, req_u->gag_u->wnd_w / 2);
     req_u->gag_u->wnd_w = req_u->gag_u->sst_w;
     req_u->gag_u->rto_w = _clamp_rto(req_u->gag_u->rto_w * 2);
   }
@@ -1159,7 +1159,7 @@ _update_resend_timer(u3_pend_req *req_u)
   //               0 :
   //               now_d - req_u->wat_u[idx_d].sen_d;
   c3_d next_expiry = req_u->gag_u->rto_w;
-  u3l_log("next_expiry %llu", next_expiry / 1000);
+  // u3l_log("next_expiry %llu", next_expiry / 1000);
   uv_timer_start(&req_u->tim_u, _mesa_packet_timeout, next_expiry / 1000, 0);
 }
 
@@ -1189,11 +1189,11 @@ _mesa_burn_misorder_queue(u3_pend_req* req_u)
       break;
     }
   }
-  // ratchet forward
-  u3l_log("updating lef_d %u", req_u->lef_d);
-  req_u->lef_d += num_w;  //  XX could we be skipping some lef_d's that we have not received?
-  u3l_log("updated lef_d %u num_w: %u", req_u->lef_d, num_w);
 
+  // ratchet forward
+  num_w++; // account for the in-ordered packet processed in _mesa_req_pact_done
+  req_u->lef_d += num_w;
+  req_u->hav_d += num_w;
   memset(req_u->mis_u, 0, num_w * sizeof(u3_misord_buf));
   memcpy(req_u->mis_u,
          (c3_y*)req_u->mis_u + (num_w * sizeof(u3_misord_buf)),
@@ -1227,7 +1227,7 @@ _mesa_req_pact_done(u3_pend_req*  req_u,
 
   // received duplicate
   if ( c3n == bitset_has(&req_u->was_u, nam_u->fra_d) ) {
-    MESA_LOG(sam_u, DUPE);
+    // MESA_LOG(sam_u, DUPE);
     return;
   }
 
@@ -1241,25 +1241,22 @@ _mesa_req_pact_done(u3_pend_req*  req_u,
 
   if ( req_u->los_u->counter != nam_u->fra_d ) {
     if ( nam_u->fra_d < req_u->los_u->counter ) {
-      u3l_log("fragment number too low: %"PRIu64, nam_u->fra_d);
+      // u3l_log("fragment number too low: %"PRIu64, nam_u->fra_d);
       c3_free(par_u);
       return;
     } else if ( nam_u->fra_d >= req_u->los_u->counter + (sizeof(req_u->mis_u)/sizeof(u3_misord_buf)) ) {
-      u3l_log("fragment number too high: %"PRIu64, nam_u->fra_d);
-      _mesa_burn_misorder_queue(req_u);
+      // u3l_log("fragment number too high: %"PRIu64, nam_u->fra_d);
       c3_free(par_u);
       return;
-    } //else { // XXXX
-    //   // insert into misordered queue
-    //                                       //  (4 - 3) - 1
-    //   u3_misord_buf* buf_u = &req_u->mis_u[nam_u->fra_d - req_u->los_u->counter - 1];
-    //   buf_u->fra_y = c3_calloc(dat_u->len_w);
-    //   buf_u->len_w = dat_u->len_w;
-    //   memcpy(buf_u->fra_y, dat_u->fra_y, dat_u->len_w);
-    //   buf_u->par_u = par_u;
-    //   u3l_log("insert into misordered queue fra: %llu",  nam_u->fra_d );
-    //   bitset_del(&req_u->was_u, nam_u->fra_d);
-    // }
+    } else {
+      // insert into misordered queue
+      u3_misord_buf* buf_u = &req_u->mis_u[nam_u->fra_d - req_u->los_u->counter - 1];
+      buf_u->fra_y = c3_calloc(dat_u->len_w);
+      buf_u->len_w = dat_u->len_w;
+      memcpy(buf_u->fra_y, dat_u->fra_y, dat_u->len_w);
+      buf_u->par_u = par_u;
+      u3l_log("insert into misordered queue fra: [%llu] = %llu",  nam_u->fra_d - req_u->los_u->counter - 1, nam_u->fra_d );
+    }
   }
   else if ( c3y != lss_verifier_ingest(req_u->los_u, dat_u->fra_y, dat_u->len_w, par_u) ) {
     u3l_log("auth fail frag %"PRIu64, nam_u->fra_d);
@@ -1275,7 +1272,7 @@ _mesa_req_pact_done(u3_pend_req*  req_u,
     return;
   }
   else {
-    u3l_log("about to other free");
+    // u3l_log("about to other free");
     c3_free(par_u);
   }
 
@@ -1283,19 +1280,13 @@ _mesa_req_pact_done(u3_pend_req*  req_u,
     req_u->ack_d = nam_u->fra_d;
   }
 
-  #ifdef MESA_DEBUG
-    if ( nam_u->fra_d != 0 && req_u->wat_u[nam_u->fra_d].tie_y != 1 ) {
-      u3l_log("received retry %"PRIu64, nam_u->fra_d);
-    }
-  #endif
-
-  req_u->hav_d++;
+  bitset_del(&req_u->was_u, nam_u->fra_d);
 
   #ifdef MESA_DEBUG
-    u3l_log("fragment %llu hav_d %llu nex_d %llu ack_d %llu lef_d %llu old_d %llu", nam_u->fra_d, req_u->hav_d, req_u->nex_d, req_u->ack_d, req_u->lef_d, req_u->old_d);
+    u3l_log("fragment %llu counter %llu hav_d %llu nex_d %llu ack_d %llu lef_d %llu old_d %llu", nam_u->fra_d, req_u->los_u->counter, req_u->hav_d, req_u->nex_d, req_u->ack_d, req_u->lef_d, req_u->old_d);
   #endif
   if ( req_u->lef_d == nam_u->fra_d ) {
-    bitset_del(&req_u->was_u, nam_u->fra_d);
+    req_u->hav_d++;
     req_u->lef_d++;
   }
 
@@ -1878,7 +1869,7 @@ _mesa_get_jumbo_cache(u3_mesa* sam_u, u3_mesa_name* nam_u)
   u3_noun pax = _name_to_jumbo_scry(nam_u);
   u3_weak res = u3h_get(sam_u->pac_p, pax);
   #ifdef MESA_DEBUG
-    u3m_p((u3_none == res)? "mesa: cache miss" : "mesa: cache hit ", pax);
+    // u3m_p((u3_none == res)? "mesa: cache miss" : "mesa: cache hit ", pax);
   #endif
   u3z(pax);
   return ( u3_none == res ) ? NULL : u3a_into(res);
@@ -1936,9 +1927,9 @@ _mesa_send_leaf(u3_mesa*      sam_u,
   u3_weak pin = _mesa_get_pit(sam_u, nam_u);
   if ( u3_none != pin) {
     #ifdef MESA_DEBUG
-      u3l_log(" sending leaf packet, fra_d: %"PRIu64, nam_u->fra_d);
+      // u3l_log(" sending leaf packet, fra_d: %"PRIu64, nam_u->fra_d);
     #endif
-    log_pact(pac_u);
+    // log_pact(pac_u);
     _mesa_send_pact(sam_u, u3k(u3t(pin)), NULL, pac_u);
     _mesa_del_pit(sam_u, nam_u);
     u3z(pin);
@@ -1949,7 +1940,7 @@ static void
 _mesa_send_jumbo_pieces(u3_mesa* sam_u, u3_mesa_line* lin_u, c3_d* fra_u)
 {
   #ifdef MESA_DEBUG
-    u3l_log("mesa: send_jumbo_pieces()");
+    // u3l_log("mesa: send_jumbo_pieces()");
   #endif
 
   u3_mesa_pact pac_u = {0};
@@ -2054,8 +2045,8 @@ _mesa_page_scry_jumbo_cb(void* vod_p, u3_noun res)
   }
 
   #ifdef MESA_DEBUG
-    u3l_log("mesa: scry_jumbo_cb()");
-    log_pact(pac_u);
+    // u3l_log("mesa: scry_jumbo_cb()");
+    // log_pact(pac_u);
   #endif
 
   u3_mesa_line* lin_u;
@@ -2064,7 +2055,6 @@ _mesa_page_scry_jumbo_cb(void* vod_p, u3_noun res)
     c3_y* jumbo_y = c3_calloc(jumbo_w);
     u3r_bytes(0, jumbo_w, jumbo_y, pac);
 
-    u3l_log("sifting jumbo packet len=%u", jumbo_w);
     u3_mesa_pact jum_u;
     c3_c* err_c = mesa_sift_pact_from_buf(&jum_u, jumbo_y, jumbo_w);
     if ( err_c ) {
@@ -2278,7 +2268,6 @@ _mesa_req_pact_init(u3_mesa* sam_u, u3_mesa_pict* pic_u, u3_lane* lan_u)
   lss_root(root, pof_u, pof_w);
   req_u->los_u = c3_calloc(sizeof(lss_verifier));
   lss_verifier_init(req_u->los_u, 0, req_u->tof_d, pof_u);
-  u3l_log("about to c3_free(pof_u)");
   c3_free(pof_u);
 
   req_u = _mesa_put_request(sam_u, nam_u, req_u);
@@ -2390,8 +2379,8 @@ static void
 _mesa_hear_page(u3_mesa_pict* pic_u, u3_lane lan_u)
 {
   #ifdef MESA_DEBUG
-    u3l_log("mesa: hear_page()");
-    log_pact(&pic_u->pac_u);
+    // u3l_log("mesa: hear_page()");
+    // log_pact(&pic_u->pac_u);
     u3_assert( PACT_PAGE == pic_u->pac_u.hed_u.typ_y );
   #endif
 
@@ -2435,7 +2424,8 @@ _mesa_hear_page(u3_mesa_pict* pic_u, u3_lane lan_u)
 
   if ( u3_none == pin ) {
     #ifdef MESA_DEBUG
-      u3l_log(" no PIT entry");
+      // u3l_log(" no PIT entry");
+      // log_name(nam_u);
     #endif
     return;
   }
@@ -2537,7 +2527,7 @@ _mesa_hear_page(u3_mesa_pict* pic_u, u3_lane lan_u)
   if ( c3y == done_with_jumbo_frame ) {
     u3_noun cad;
 
-    u3l_log(" received last packet, tot_d: %llu", req_u->tof_d);
+    u3l_log(" received last packet, tof_d: %llu tob_d: %llu", req_u->tof_d, req_u->tob_d);
     c3_d now_d = _get_now_micros();
     u3l_log("%llu kilobytes took %f ms", req_u->tof_d, (now_d - sam_u->tim_d)/1000.0);
 
@@ -2606,7 +2596,7 @@ static void
 _mesa_hear_peek(u3_mesa_pict* pic_u, u3_lane lan_u)
 {
   #ifdef MESA_DEBUG
-    u3l_log("mesa: hear_peek()");
+    // u3l_log("mesa: hear_peek()");
     u3_assert( PACT_PEEK == pic_u->pac_u.hed_u.typ_y );
   #endif