fix: compendium of fixes to replay and firedancer leader pipeline

contrib/test/setup_fd_cluster.sh

@@ -28,7 +28,7 @@ GENESIS_OUTPUT=$(./agave/target/release/solana-genesis \
     --bootstrap-validator test-ledger/validator-keypair.json test-ledger/vote-account-keypair.json test-ledger/stake-account-keypair.json \
     --bootstrap-stake-authorized-pubkey test-ledger/validator-keypair.json \
     --bootstrap-validator-lamports 11000000000000000 \
-    --bootstrap-validator-stake-lamports 10000000000000000 \
+    --bootstrap-validator-stake-lamports 1000000000000000 \
     --faucet-pubkey test-ledger/faucet-keypair.json --faucet-lamports 1000000000000000000 \
     --slots-per-epoch 200 \
     --hashes-per-tick 128 \
@@ -45,7 +45,7 @@ _PRIMARY_INTERFACE=$(ip route show default | awk '/default/ {print $5}')
 RUST_LOG=debug ./agave/target/release/agave-validator \
     --identity test-ledger/validator-keypair.json \
     --ledger test-ledger \
-    --limit-ledger-size 100000000 \
+    --limit-ledger-size 1000000000 \
     --no-genesis-fetch \
     --no-snapshot-fetch \
     --no-poh-speed-test \
@@ -64,4 +64,8 @@ RUST_LOG=debug ./agave/target/release/agave-validator \
     --full-rpc-api \
     --allow-private-addr \
     --rpc-faucet-address 127.0.0.1:9900 \
-    --log test-ledger/validator.log
+    --enable-rpc-transaction-history \
+    --tpu-enable-udp \
+    --log test-ledger/validator.log
+    # --tpu-disable-quic \
+    # --public-tpu-address 127.0.0.1:9010 \

contrib/test/setup_fd_cluster_stakes.sh

@@ -5,18 +5,32 @@ IFS=$'\n\t'
 cd ../test-ledger
 
 PRIMARY_IP=$(ip -o -4 addr show scope global | awk '{ print $4 }' | cut -d/ -f1)
-RPC_URL="http://$PRIMARY_IP:8899/"
+# RPC_URL="http:/n/$PRIMARY_IP:8899/"
 RPC_URL="http://localhost:8899/"
 
 solana-keygen new --no-bip39-passphrase --silent --outfile fd-identity-keypair.json
 solana-keygen new --no-bip39-passphrase --silent --outfile fd-stake-keypair.json
 solana-keygen new --no-bip39-passphrase --silent --outfile fd-vote-keypair.json
 solana-keygen new --no-bip39-passphrase --silent --outfile fd-withdrawer-keypair.json
 
-solana -u $RPC_URL --keypair faucet-keypair.json transfer --allow-unfunded-recipient fd-identity-keypair.json 4000000
+solana -u $RPC_URL --keypair faucet-keypair.json transfer --allow-unfunded-recipient fd-identity-keypair.json 400000
 solana -u $RPC_URL --keypair fd-identity-keypair.json create-vote-account fd-vote-keypair.json fd-identity-keypair.json fd-withdrawer-keypair.json
-solana -u $RPC_URL --keypair fd-identity-keypair.json create-stake-account fd-stake-keypair.json 3000000
+solana -u $RPC_URL --keypair fd-identity-keypair.json create-stake-account fd-stake-keypair.json 300000
 solana -u $RPC_URL --keypair fd-identity-keypair.json delegate-stake fd-stake-keypair.json fd-vote-keypair.json
 
 solana -u $RPC_URL --keypair fd-identity-keypair.json vote-account fd-vote-keypair.json
 solana -u $RPC_URL --keypair fd-identity-keypair.json stake-account fd-stake-keypair.json
+
+exit 1
+solana-keygen new --no-bip39-passphrase --silent --outfile fd-identity-keypair-2.json
+solana-keygen new --no-bip39-passphrase --silent --outfile fd-stake-keypair-2.json
+solana-keygen new --no-bip39-passphrase --silent --outfile fd-vote-keypair-2.json
+solana-keygen new --no-bip39-passphrase --silent --outfile fd-withdrawer-keypair-2.json
+
+solana -u $RPC_URL --keypair faucet-keypair.json transfer --allow-unfunded-recipient fd-identity-keypair-2.json 4000000
+solana -u $RPC_URL --keypair fd-identity-keypair-2.json create-vote-account fd-vote-keypair-2.json fd-identity-keypair-2.json fd-withdrawer-keypair-2.json
+solana -u $RPC_URL --keypair fd-identity-keypair-2.json create-stake-account fd-stake-keypair-2.json 100000
+solana -u $RPC_URL --keypair fd-identity-keypair-2.json delegate-stake fd-stake-keypair-2.json fd-vote-keypair-2.json
+
+solana -u $RPC_URL --keypair fd-identity-keypair-2.json vote-account fd-vote-keypair-2.json
+solana -u $RPC_URL --keypair fd-identity-keypair-2.json stake-account fd-stake-keypair-2.json

contrib/test/test_firedancer_leader.sh

@@ -35,8 +35,9 @@ FULL_SNAPSHOT=$(wget -c -nc -S --trust-server-names http://$PRIMARY_IP:8899/snap
 echo "
 name = \"fd1test\"
 [layout]
-    affinity = \"1-32\"
+    affinity = \"1-37\"
     bank_tile_count = 1
+    verify_tile_count = 16
 [gossip]
     port = 8700
 [tiles]

contrib/tool/block_production.py

@@ -0,0 +1,55 @@
+from solders.message import Message
+from solders.keypair import Keypair
+# from solders.instruction import Instruction
+# from solders.hash import Hash
+from solana.transaction import Transaction
+from solders.system_program import TransferParams, transfer
+from solders.pubkey import Pubkey
+
+import datetime
+import sys
+import socket
+import time
+import tqdm
+import itertools
+from multiprocessing import Pool, TimeoutError
+from functools import partial
+from pqdm.processes import pqdm
+
+from solana.rpc.api import Client
+import random
+
+def usage():
+  print("contrib/tool/block_production.py <rpc_url> <pubkey> <n>")
+  exit(1)
+
+def main():
+  if len(sys.argv) < 4:
+    usage()
+
+  client = Client(sys.argv[1])
+  pubkey = Pubkey.from_string(sys.argv[2])
+  slot_cnt = int(sys.argv[3])
+
+  epoch_info = client.get_epoch_info().value
+  leader_schedule = client.get_leader_schedule(epoch_info.absolute_slot)
+  leader_slot_idxs = leader_schedule.value[pubkey]
+  upcoming_slot_idxs = list(filter(lambda s: s > epoch_info.slot_index, leader_slot_idxs))[:4*slot_cnt]
+  epoch_start_slot = epoch_info.absolute_slot-epoch_info.slot_index
+
+  prev_slot = epoch_info.absolute_slot
+  for slot_idx in upcoming_slot_idxs:
+    slot = epoch_start_slot+slot_idx
+    rel = slot-prev_slot
+    if prev_slot+1==slot:
+      prev_slot = slot  
+      continue
+    dist = slot-epoch_info.absolute_slot
+    prev_slot = slot
+    rel_td = datetime.timedelta(milliseconds=rel*400)
+    dist_td = datetime.timedelta(milliseconds=dist*400)
+    print("slot: {:<10}| rel: {:>8}| dist: {:>8}| time(s): {:<16} ({})".format(slot, rel, dist, str(dist_td), str(rel_td)))
+
+
+if __name__ == '__main__':
+  main()

src/app/fdctl/run/tiles/fd_gossip.c

@@ -308,6 +308,22 @@ gossip_deliver_fun( fd_crds_data_t * data, void * arg ) {
     if (ele) {
       ele->contact_info = *contact_info;
     }
+  } else if( fd_crds_data_is_contact_info_v2( data ) ) {
+    fd_gossip_contact_info_v2_t const * contact_info_v2 = &data->inner.contact_info_v2;
+
+    fd_gossip_contact_info_v1_t contact_info;
+    fd_gossip_contact_info_v2_to_v1( contact_info_v2, &contact_info );
+    FD_LOG_DEBUG(("contact info v2 - ip: " FD_IP4_ADDR_FMT ", port: %u", FD_IP4_ADDR_FMT_ARGS( contact_info.gossip.addr.inner.ip4 ), contact_info.gossip.port ));
+
+    fd_contact_info_elem_t * ele = fd_contact_info_table_query( ctx->contact_info_table, &contact_info.id, NULL );
+    if (FD_UNLIKELY(!ele &&
+                    !fd_contact_info_table_is_full(ctx->contact_info_table))) {
+      ele = fd_contact_info_table_insert(ctx->contact_info_table,
+                                         &contact_info.id);
+    }
+    if (ele) {
+      ele->contact_info = contact_info;
+    }
   }
 }
 
@@ -498,6 +514,7 @@ after_credit( void *             _ctx,
 
     ulong tspub = fd_frag_meta_ts_comp( fd_tickcount() );
 
+    FD_LOG_INFO(( "publishing peers - tvu: %lu, repair: %lu, tpu_vote: %lu", tvu_peer_cnt, repair_peers_cnt, voter_peers_cnt ));
     if( tvu_peer_cnt>0 ) {
       *shred_dest_msg         = tvu_peer_cnt;
       ulong shred_contact_sz  = sizeof(ulong) + (tvu_peer_cnt * sizeof(fd_shred_dest_wire_t));