flamenco, runtime: re-work partitioned epoch rewards

src/flamenco/capture/fd_solcap_proto.h

@@ -142,7 +142,9 @@ typedef struct fd_solcap_account_tbl fd_solcap_account_tbl_t;
 #define FD_SOLCAP_FHDR_SZ (256UL)
 
 /* FD_SOLCAP_ACC_TBL_CNT is the number of entries that fit in the in-
-   memory buffer for the account table. */
+   memory buffer for the account table.
+   
+   N.b: to support epoch boundaries increase this number to 2097152 */
 
 #define FD_SOLCAP_ACC_TBL_CNT (8192U)
 

src/flamenco/capture/fd_solcap_writer.c

@@ -196,11 +196,8 @@ fd_solcap_writer_init( fd_solcap_writer_t * writer,
   }
 
   /* Init writer */
-
-  *writer = (fd_solcap_writer_t) {
-    .file        = (FILE *)file,
-    .stream_goff = (ulong)stream_goff
-  };
+  writer->file        = file;
+  writer->stream_goff = stream_goff;
 
   return writer;
 }
@@ -285,6 +282,7 @@ fd_solcap_flush_account_table( fd_solcap_writer_t * writer ) {
 
   /* FIXME: This breaks account recording for epoch boundaries and needs to be fixed */
   if( writer->account_idx >= FD_SOLCAP_ACC_TBL_CNT ) {
+    FD_LOG_WARNING(( "too many records in solcap accounts table" ));
     writer->account_idx = 0UL;
     return 0;
   }

src/flamenco/rewards/fd_rewards.h

@@ -14,26 +14,23 @@ FD_PROTOTYPES_BEGIN
 
 void
 fd_update_rewards( fd_exec_slot_ctx_t * slot_ctx,
+                   const fd_hash_t *    blockhash,
                    ulong                prev_epoch );
 
 void
-fd_begin_partitioned_rewards( fd_exec_slot_ctx_t * slot_ctx,
-                              ulong                parent_epoch );
+fd_begin_partitioned_rewards(
+                    fd_exec_slot_ctx_t * slot_ctx,
+                    const fd_hash_t *    blockhash,
+                    ulong                parent_epoch );
 
 void
-fd_distribute_partitioned_epoch_rewards( fd_exec_slot_ctx_t * slot_ctx );
-
-struct fd_inflation_rates {
-    ulong epoch;
-    double foundation;
-    double total;
-    double validator;
-};
-typedef struct fd_inflation_rates fd_inflation_rates_t;
+fd_rewards_recalculate_partitioned_rewards(
+    fd_exec_slot_ctx_t * slot_ctx,
+    const fd_hash_t * parent_blockhash
+);
 
 void
-fd_calculate_inflation_rates( fd_exec_slot_ctx_t *   slot_ctx,
-                              fd_inflation_rates_t * rates );
+fd_distribute_partitioned_epoch_rewards( fd_exec_slot_ctx_t * slot_ctx );
 
 FD_PROTOTYPES_END
 

src/flamenco/rewards/fd_rewards_types.h

@@ -16,106 +16,16 @@
 #undef VECT_NAME
 #undef VECT_ELEMENT
 
-/* reward calculation happens synchronously during the first block of the epoch boundary.
- So, # blocks for reward calculation is 1. */
-#define REWARD_CALCULATION_NUM_BLOCK            1
-/* stake accounts to store in one block during partitioned reward interval Target to store 64 rewards per entry/tick in a block. A block has a minimum of 64 entries/tick. This gives 4096 total rewards to store in one block. This constant affects consensus. */
-#define STAKE_ACCOUNT_STORES_PER_BLOCK          4096
-#define TEST_ENABLE_PARTITIONED_REWARDS         0
-#define TEST_COMPARE_PARTITIONED_EPOCH_REWARDS  0
-#define MAX_FACTOR_OF_REWARD_BLOCKS_IN_EPOCH    10
-
-struct fd_vote_reward_t_mapnode {
-  fd_pubkey_t vote_pubkey;
-  ulong vote_rewards;
-  uchar commission;
-  uchar needs_store;
-};
-
-typedef struct fd_vote_reward_t_mapnode fd_vote_reward_t_mapnode_t;
-
-#define MAP_NAME              fd_vote_reward_t_map
-#define MAP_T                 fd_vote_reward_t_mapnode_t
-#define MAP_MEMOIZE           0
-#define MAP_KEY               vote_pubkey
-#define MAP_KEY_T             fd_pubkey_t
-#define MAP_KEY_NULL          (fd_pubkey_t){0}
-#define MAP_KEY_INVAL(k)      MAP_KEY_EQUAL((k),MAP_KEY_NULL)
-#define MAP_KEY_EQUAL(k0,k1)    (!memcmp((k0).key, (k1).key, sizeof( fd_pubkey_t ) ))
-#define MAP_KEY_EQUAL_IS_SLOW 1
-#define MAP_KEY_HASH(key)     ((uint)fd_ulong_hash( fd_ulong_load_8( (key).key ) ))
-#define MAP_KEY_MOVE(kd,ks) memcpy( &(kd), &(ks),sizeof(fd_pubkey_t))
-#include "../../util/tmpl/fd_map_dynamic.c"
-static inline fd_vote_reward_t_mapnode_t *
-fd_vote_reward_t_map_alloc( fd_valloc_t valloc, int lg_slot_cnt ) {
-  void * mem = fd_valloc_malloc( valloc, fd_vote_reward_t_map_align(), fd_vote_reward_t_map_footprint( lg_slot_cnt ));
-  return fd_vote_reward_t_map_join(fd_vote_reward_t_map_new(mem, lg_slot_cnt));
-}
-
-
-#define DEQUE_NAME deq_fd_stake_reward_t
-#define DEQUE_T    fd_stake_reward_t
-#define DEQUE_MAX  2000000UL
-#include "../../util/tmpl/fd_deque.c"
-static inline fd_stake_reward_t *
-deq_fd_stake_reward_t_alloc( fd_valloc_t valloc ) {
-  void * mem = fd_valloc_malloc( valloc, deq_fd_stake_reward_t_align(), deq_fd_stake_reward_t_footprint());
-  return deq_fd_stake_reward_t_join( deq_fd_stake_reward_t_new( mem ) );
-}
-
-struct fd_validator_reward_calculation {
-    fd_acc_lamports_t total_stake_rewards_lamports;
-    fd_stake_reward_t * stake_reward_deq;
-    fd_vote_reward_t_mapnode_t * vote_reward_map;
-};
-typedef struct fd_validator_reward_calculation fd_validator_reward_calculation_t;
+/* Number of blocks for reward calculation and storing vote accounts.
+   Distributing rewards to stake accounts begins AFTER this many blocks.
+   
+   https://github.com/anza-xyz/agave/blob/9a7bf72940f4b3cd7fc94f54e005868ce707d53d/runtime/src/bank/partitioned_epoch_rewards/mod.rs#L27 */
+#define REWARD_CALCULATION_NUM_BLOCKS ( 1UL )
 
-struct fd_partitioned_rewards_calculation {
-    /* VoteRewardsAccount */
-    fd_vote_reward_t_mapnode_t * vote_account_rewards;
-    fd_stake_rewards_vector_t stake_rewards_by_partition[1];
-    ulong total_stake_rewards_lamports;
-    ulong old_vote_balance_and_staked;
-    ulong validator_rewards;
-    double validator_rate;
-    double foundation_rate;
-    double prev_epoch_duration_in_years;
-    ulong capitalization;
-};
-typedef struct fd_partitioned_rewards_calculation fd_partitioned_rewards_calculation_t;
-
-struct fd_point_value {
-  ulong rewards;
-  uint128 points;
-};
-typedef struct fd_point_value fd_point_value_t;
-
-struct fd_calculated_stake_rewards{
-  ulong staker_rewards;
-  ulong voter_rewards;
-  ulong new_credits_observed;
-};
-typedef struct fd_calculated_stake_rewards fd_calculated_stake_rewards_t;
-
-struct fd_calculate_stake_points {
-  uint128 points;
-  ulong new_credits_observed;
-  uint force_credits_update_with_skipped_reward;
-};
-typedef struct fd_calculate_stake_points fd_calculate_stake_points_t;
-
-struct fd_calculate_rewards_and_distribute_vote_rewards_result {
-  ulong total_rewards;
-  ulong distributed_rewards;
-  fd_stake_rewards_vector_t stake_rewards_by_partition[1];
-};
-typedef struct fd_calculate_rewards_and_distribute_vote_rewards_result fd_calculate_rewards_and_distribute_vote_rewards_result_t;
-
-struct fd_epoch_reward_status {
-  uint is_active;
-  ulong start_block_height;
-  fd_stake_rewards_vector_t stake_rewards_by_partition[1];
-};
-typedef struct fd_epoch_reward_status fd_epoch_reward_status_t;
+/* stake accounts to store in one block during partitioned reward interval Target to store 64 rewards per entry/tick in a block. A block has a minimum of 64 entries/tick. This gives 4096 total rewards to store in one block. This constant affects consensus. */
+#define STAKE_ACCOUNT_STORES_PER_BLOCK          ( 4096UL )
+#define TEST_ENABLE_PARTITIONED_REWARDS         ( 0UL )
+#define TEST_COMPARE_PARTITIONED_EPOCH_REWARDS  ( 0UL )
+#define MAX_FACTOR_OF_REWARD_BLOCKS_IN_EPOCH    ( 10UL ) 
 
 #endif /* HEADER_fd_src_flamenco_runtime_program_fd_rewards_types_h */

src/flamenco/runtime/context/fd_exec_slot_ctx.c

@@ -28,6 +28,9 @@ fd_exec_slot_ctx_new( void *      mem,
   self->sysvar_cache = fd_sysvar_cache_new( fd_valloc_malloc( valloc, fd_sysvar_cache_align(), fd_sysvar_cache_footprint() ), valloc );
   self->account_compute_table = fd_account_compute_table_join( fd_account_compute_table_new( fd_valloc_malloc( valloc, fd_account_compute_table_align(), fd_account_compute_table_footprint( 10000 ) ), 10000, 0 ) );
 
+  /* This is inactive by default */
+  self->epoch_reward_status.discriminant = fd_epoch_reward_status_enum_Inactive;
+
   FD_COMPILER_MFENCE();
   self->magic = FD_EXEC_SLOT_CTX_MAGIC;
   FD_COMPILER_MFENCE();
@@ -280,6 +283,8 @@ fd_exec_slot_ctx_recover_( fd_exec_slot_ctx_t *   slot_ctx,
     fd_memset( &stakes1->stakes.vote_accounts, 0, sizeof(fd_vote_accounts_t) );
   } while(0);
 
+  /* FIXME: handle epoch reward status and serialization */
+
   // TODO Backup to database
   //int result = fd_runtime_save_epoch_bank(slot_ctx);
   //if( result != FD_EXECUTOR_INSTR_SUCCESS ) {
@@ -394,6 +399,7 @@ fd_exec_slot_ctx_free( fd_exec_slot_ctx_t * slot_ctx ) {
 
   /* leader points to a caller-allocated leader schedule */
 
-  fd_stake_rewards_vector_destroy( slot_ctx->epoch_reward_status.stake_rewards_by_partition );
+  /* FIXME: clean this up */
+  // fd_stake_rewards_vector_destroy( slot_ctx->epoch_reward_status.stake_rewards_by_partition );
   fd_exec_slot_ctx_delete( fd_exec_slot_ctx_leave( slot_ctx ) );
 }

src/flamenco/runtime/fd_executor.c

@@ -275,7 +275,7 @@ fd_executor_check_txn_accounts( fd_exec_txn_ctx_t * txn_ctx ) {
       validated_fee_payer = 1;
     }
 
-    if( txn_ctx->slot_ctx->epoch_reward_status.is_active && fd_txn_account_is_writable_idx( txn_ctx, (int)i )
+    if( fd_epoch_reward_status_is_Active( &txn_ctx->slot_ctx->epoch_reward_status ) && fd_txn_account_is_writable_idx( txn_ctx, (int)i )
         && memcmp( acct->const_meta->info.owner, fd_solana_stake_program_id.uc, sizeof(fd_pubkey_t))==0 ) {
       return FD_RUNTIME_TXN_ERR_PROGRAM_EXECUTION_TEMPORARILY_RESTRICTED;
     }

src/flamenco/runtime/fd_runtime.c

@@ -903,6 +903,7 @@ fd_runtime_prepare_txns_phase2_tpool( fd_exec_slot_ctx_t * slot_ctx,
 
         if ( fd_hash_hash_age_pair_t_map_find( slot_ctx->slot_bank.block_hash_queue.ages_pool, slot_ctx->slot_bank.block_hash_queue.ages_root, &key ) == NULL ) {
           task_info[ txn_idx ].txn->flags = 0;
+          FD_LOG_WARNING(( "blockhash not found" ));
           res |= FD_RUNTIME_TXN_ERR_BLOCKHASH_NOT_FOUND;
           continue;
         }
@@ -921,7 +922,7 @@ fd_runtime_prepare_txns_phase2_tpool( fd_exec_slot_ctx_t * slot_ctx,
         // TODO: figure out if it is faster to batch query properly and loop all txns again
         fd_txncache_query_batch( slot_ctx->status_cache, &curr_query, 1UL, query_arg, query_func, &err );
         if( err != FD_RUNTIME_EXECUTE_SUCCESS ) {
-          // FD_LOG_WARNING(("HELLO %64J", (uchar *)txn_ctx->_txn_raw->raw + txn_ctx->txn_descriptor->signature_off));
+          FD_LOG_WARNING(( "txncache_query_batch failed %64J", (uchar *)txn_ctx->_txn_raw->raw + txn_ctx->txn_descriptor->signature_off ));
           task_info[ txn_idx ].txn->flags = 0;
           res |= FD_RUNTIME_TXN_ERR_ALREADY_PROCESSED;
           continue;
@@ -931,6 +932,7 @@ fd_runtime_prepare_txns_phase2_tpool( fd_exec_slot_ctx_t * slot_ctx,
       err = fd_executor_check_txn_accounts( txn_ctx );
       if ( err != FD_RUNTIME_EXECUTE_SUCCESS ) {
         task_info[ txn_idx ].txn->flags = 0;
+        FD_LOG_WARNING(( "fd_executor_check_txn_accounts failed" ));
         res |= err;
         continue;
       }
@@ -3709,9 +3711,9 @@ void fd_process_new_epoch(
            "update_epoch_stakes",
        ); */
   if ( FD_FEATURE_ACTIVE( slot_ctx, enable_partitioned_epoch_reward ) ) {
-    fd_begin_partitioned_rewards( slot_ctx, parent_epoch );
+    fd_begin_partitioned_rewards( slot_ctx, &slot_ctx->slot_bank.banks_hash, parent_epoch );
   } else {
-    fd_update_rewards( slot_ctx, parent_epoch );
+    fd_update_rewards( slot_ctx, &slot_ctx->slot_bank.banks_hash, parent_epoch );
   }
 
   fd_update_stake_delegations( slot_ctx );

src/flamenco/runtime/program/fd_stake_program.c

@@ -2399,7 +2399,7 @@ fd_stake_program_execute( fd_exec_instr_ctx_t ctx ) {
      enable_partitioned_epoch_reward feature is activated. If it exists, check
      the `active` field */
   fd_sysvar_epoch_rewards_t const * rewards = fd_sysvar_cache_epoch_rewards( ctx.slot_ctx->sysvar_cache );
-  int epoch_rewards_active = (NULL != rewards) ? rewards->epoch_rewards.active : false;
+  int epoch_rewards_active = (NULL != rewards) ? rewards->active : false;
 
   if (epoch_rewards_active && instruction->discriminant != fd_stake_instruction_enum_get_minimum_delegation) {
     ctx.txn_ctx->custom_err = FD_STAKE_ERR_EPOCH_REWARDS_ACTIVE;