flamenco, runtime: re-work partitioned epoch rewards

src/flamenco/rewards/fd_rewards.h

@@ -14,27 +14,18 @@ FD_PROTOTYPES_BEGIN
 
 void
 fd_update_rewards( fd_exec_slot_ctx_t * slot_ctx,
+                   fd_hash_t *          blockhash,
                    ulong                prev_epoch );
 
 void
-fd_begin_partitioned_rewards( fd_exec_slot_ctx_t * slot_ctx,
-                              ulong                parent_epoch );
+fd_begin_partitioned_rewards(
+                    fd_exec_slot_ctx_t * slot_ctx,
+                    fd_hash_t * blockhash,
+                    ulong parent_epoch );
 
 void
 fd_distribute_partitioned_epoch_rewards( fd_exec_slot_ctx_t * slot_ctx );
 
-struct fd_inflation_rates {
-    ulong epoch;
-    double foundation;
-    double total;
-    double validator;
-};
-typedef struct fd_inflation_rates fd_inflation_rates_t;
-
-void
-fd_calculate_inflation_rates( fd_exec_slot_ctx_t *   slot_ctx,
-                              fd_inflation_rates_t * rates );
-
 FD_PROTOTYPES_END
 
 #endif

src/flamenco/rewards/fd_rewards_types.h

@@ -16,106 +16,16 @@
 #undef VECT_NAME
 #undef VECT_ELEMENT
 
-/* reward calculation happens synchronously during the first block of the epoch boundary.
- So, # blocks for reward calculation is 1. */
-#define REWARD_CALCULATION_NUM_BLOCK            1
-/* stake accounts to store in one block during partitioned reward interval Target to store 64 rewards per entry/tick in a block. A block has a minimum of 64 entries/tick. This gives 4096 total rewards to store in one block. This constant affects consensus. */
-#define STAKE_ACCOUNT_STORES_PER_BLOCK          4096
-#define TEST_ENABLE_PARTITIONED_REWARDS         0
-#define TEST_COMPARE_PARTITIONED_EPOCH_REWARDS  0
-#define MAX_FACTOR_OF_REWARD_BLOCKS_IN_EPOCH    10
-
-struct fd_vote_reward_t_mapnode {
-  fd_pubkey_t vote_pubkey;
-  ulong vote_rewards;
-  uchar commission;
-  uchar needs_store;
-};
-
-typedef struct fd_vote_reward_t_mapnode fd_vote_reward_t_mapnode_t;
-
-#define MAP_NAME              fd_vote_reward_t_map
-#define MAP_T                 fd_vote_reward_t_mapnode_t
-#define MAP_MEMOIZE           0
-#define MAP_KEY               vote_pubkey
-#define MAP_KEY_T             fd_pubkey_t
-#define MAP_KEY_NULL          (fd_pubkey_t){0}
-#define MAP_KEY_INVAL(k)      MAP_KEY_EQUAL((k),MAP_KEY_NULL)
-#define MAP_KEY_EQUAL(k0,k1)    (!memcmp((k0).key, (k1).key, sizeof( fd_pubkey_t ) ))
-#define MAP_KEY_EQUAL_IS_SLOW 1
-#define MAP_KEY_HASH(key)     ((uint)fd_ulong_hash( fd_ulong_load_8( (key).key ) ))
-#define MAP_KEY_MOVE(kd,ks) memcpy( &(kd), &(ks),sizeof(fd_pubkey_t))
-#include "../../util/tmpl/fd_map_dynamic.c"
-static inline fd_vote_reward_t_mapnode_t *
-fd_vote_reward_t_map_alloc( fd_valloc_t valloc, int lg_slot_cnt ) {
-  void * mem = fd_valloc_malloc( valloc, fd_vote_reward_t_map_align(), fd_vote_reward_t_map_footprint( lg_slot_cnt ));
-  return fd_vote_reward_t_map_join(fd_vote_reward_t_map_new(mem, lg_slot_cnt));
-}
-
-
-#define DEQUE_NAME deq_fd_stake_reward_t
-#define DEQUE_T    fd_stake_reward_t
-#define DEQUE_MAX  2000000UL
-#include "../../util/tmpl/fd_deque.c"
-static inline fd_stake_reward_t *
-deq_fd_stake_reward_t_alloc( fd_valloc_t valloc ) {
-  void * mem = fd_valloc_malloc( valloc, deq_fd_stake_reward_t_align(), deq_fd_stake_reward_t_footprint());
-  return deq_fd_stake_reward_t_join( deq_fd_stake_reward_t_new( mem ) );
-}
-
-struct fd_validator_reward_calculation {
-    fd_acc_lamports_t total_stake_rewards_lamports;
-    fd_stake_reward_t * stake_reward_deq;
-    fd_vote_reward_t_mapnode_t * vote_reward_map;
-};
-typedef struct fd_validator_reward_calculation fd_validator_reward_calculation_t;
+/* Number of blocks for reward calculation and storing vote accounts.
+   Distributing rewards to stake accounts begins AFTER this many blocks.
+   
+   https://github.com/anza-xyz/agave/blob/9a7bf72940f4b3cd7fc94f54e005868ce707d53d/runtime/src/bank/partitioned_epoch_rewards/mod.rs#L27 */
+#define REWARD_CALCULATION_NUM_BLOCKS ( 1UL )
 
-struct fd_partitioned_rewards_calculation {
-    /* VoteRewardsAccount */
-    fd_vote_reward_t_mapnode_t * vote_account_rewards;
-    fd_stake_rewards_vector_t stake_rewards_by_partition[1];
-    ulong total_stake_rewards_lamports;
-    ulong old_vote_balance_and_staked;
-    ulong validator_rewards;
-    double validator_rate;
-    double foundation_rate;
-    double prev_epoch_duration_in_years;
-    ulong capitalization;
-};
-typedef struct fd_partitioned_rewards_calculation fd_partitioned_rewards_calculation_t;
-
-struct fd_point_value {
-  ulong rewards;
-  uint128 points;
-};
-typedef struct fd_point_value fd_point_value_t;
-
-struct fd_calculated_stake_rewards{
-  ulong staker_rewards;
-  ulong voter_rewards;
-  ulong new_credits_observed;
-};
-typedef struct fd_calculated_stake_rewards fd_calculated_stake_rewards_t;
-
-struct fd_calculate_stake_points {
-  uint128 points;
-  ulong new_credits_observed;
-  uint force_credits_update_with_skipped_reward;
-};
-typedef struct fd_calculate_stake_points fd_calculate_stake_points_t;
-
-struct fd_calculate_rewards_and_distribute_vote_rewards_result {
-  ulong total_rewards;
-  ulong distributed_rewards;
-  fd_stake_rewards_vector_t stake_rewards_by_partition[1];
-};
-typedef struct fd_calculate_rewards_and_distribute_vote_rewards_result fd_calculate_rewards_and_distribute_vote_rewards_result_t;
-
-struct fd_epoch_reward_status {
-  uint is_active;
-  ulong start_block_height;
-  fd_stake_rewards_vector_t stake_rewards_by_partition[1];
-};
-typedef struct fd_epoch_reward_status fd_epoch_reward_status_t;
+/* stake accounts to store in one block during partitioned reward interval Target to store 64 rewards per entry/tick in a block. A block has a minimum of 64 entries/tick. This gives 4096 total rewards to store in one block. This constant affects consensus. */
+#define STAKE_ACCOUNT_STORES_PER_BLOCK          ( 4096UL )
+#define TEST_ENABLE_PARTITIONED_REWARDS         ( 0UL )
+#define TEST_COMPARE_PARTITIONED_EPOCH_REWARDS  ( 0UL )
+#define MAX_FACTOR_OF_REWARD_BLOCKS_IN_EPOCH    ( 10UL ) 
 
 #endif /* HEADER_fd_src_flamenco_runtime_program_fd_rewards_types_h */

src/flamenco/runtime/context/fd_exec_slot_ctx.c

@@ -394,6 +394,7 @@ fd_exec_slot_ctx_free( fd_exec_slot_ctx_t * slot_ctx ) {
 
   /* leader points to a caller-allocated leader schedule */
 
-  fd_stake_rewards_vector_destroy( slot_ctx->epoch_reward_status.stake_rewards_by_partition );
+  /* FIXME: clean this up */
+  // fd_stake_rewards_vector_destroy( slot_ctx->epoch_reward_status.stake_rewards_by_partition );
   fd_exec_slot_ctx_delete( fd_exec_slot_ctx_leave( slot_ctx ) );
 }

src/flamenco/runtime/fd_executor.c

@@ -275,7 +275,7 @@ fd_executor_check_txn_accounts( fd_exec_txn_ctx_t * txn_ctx ) {
       validated_fee_payer = 1;
     }
 
-    if( txn_ctx->slot_ctx->epoch_reward_status.is_active && fd_txn_account_is_writable_idx( txn_ctx, (int)i )
+    if( fd_epoch_reward_status_is_Active( &txn_ctx->slot_ctx->epoch_reward_status ) && fd_txn_account_is_writable_idx( txn_ctx, (int)i )
         && memcmp( acct->const_meta->info.owner, fd_solana_stake_program_id.uc, sizeof(fd_pubkey_t))==0 ) {
       return FD_RUNTIME_TXN_ERR_PROGRAM_EXECUTION_TEMPORARILY_RESTRICTED;
     }

src/flamenco/runtime/fd_runtime.c

@@ -3709,9 +3709,9 @@ void fd_process_new_epoch(
            "update_epoch_stakes",
        ); */
   if ( FD_FEATURE_ACTIVE( slot_ctx, enable_partitioned_epoch_reward ) ) {
-    fd_begin_partitioned_rewards( slot_ctx, parent_epoch );
+    fd_begin_partitioned_rewards( slot_ctx, &slot_ctx->slot_bank.banks_hash, parent_epoch );
   } else {
-    fd_update_rewards( slot_ctx, parent_epoch );
+    fd_update_rewards( slot_ctx, &slot_ctx->slot_bank.banks_hash, parent_epoch );
   }
 
   fd_update_stake_delegations( slot_ctx );

src/flamenco/runtime/program/fd_stake_program.c

@@ -2399,7 +2399,7 @@ fd_stake_program_execute( fd_exec_instr_ctx_t ctx ) {
      enable_partitioned_epoch_reward feature is activated. If it exists, check
      the `active` field */
   fd_sysvar_epoch_rewards_t const * rewards = fd_sysvar_cache_epoch_rewards( ctx.slot_ctx->sysvar_cache );
-  int epoch_rewards_active = (NULL != rewards) ? rewards->epoch_rewards.active : false;
+  int epoch_rewards_active = (NULL != rewards) ? rewards->active : false;
 
   if (epoch_rewards_active && instruction->discriminant != fd_stake_instruction_enum_get_minimum_delegation) {
     ctx.txn_ctx->custom_err = FD_STAKE_ERR_EPOCH_REWARDS_ACTIVE;

src/flamenco/runtime/sysvar/fd_sysvar_epoch_rewards.c

@@ -4,31 +4,10 @@
 #include "../fd_borrowed_account.h"
 #include "../fd_system_ids.h"
 #include "../context/fd_exec_slot_ctx.h"
-
-// THIS IS ALL WRONG... the partitioned epoch rewards code paths have not been finalized with agave
-// so these changes are here to support getting the fuzz tests to pass and not actual ledger correctness.
-//
-// Also, since this feature has not been activated in mainnet or testnet, the account itself does not
-// exist which is why they felt free to change the layout outside of a feature flag.
-//
-// Once the Agave code has stabilized, we will make a proper implementation pass
-
-void
-fd_sysvar_epoch_rewards_burn_and_purge(
-    fd_exec_slot_ctx_t  * slot_ctx
-) {
-    FD_BORROWED_ACCOUNT_DECL(rewards);
-
-    int rc = fd_acc_mgr_modify( slot_ctx->acc_mgr, slot_ctx->funk_txn, &fd_sysvar_epoch_rewards_id, 0, 0, rewards);
-    if ( FD_UNLIKELY( rc ) ) return; // not good...
-
-    fd_memcpy(rewards->meta->info.owner, fd_solana_system_program_id.key, sizeof(fd_pubkey_t));
-    rewards->meta->dlen = 0;
-    rewards->meta->info.lamports = 0;
-}
+#include "../context/fd_exec_epoch_ctx.h"
 
 static void
-write_epoch_rewards( fd_exec_slot_ctx_t * slot_ctx, fd_sysvar_epoch_rewards_t * epoch_rewards, fd_acc_lamports_t acc_lamports) {
+write_epoch_rewards( fd_exec_slot_ctx_t * slot_ctx, fd_sysvar_epoch_rewards_t * epoch_rewards ) {
   ulong sz = fd_sysvar_epoch_rewards_size( epoch_rewards );
   uchar enc[sz];
   fd_memset( enc, 0, sz );
@@ -39,20 +18,19 @@ write_epoch_rewards( fd_exec_slot_ctx_t * slot_ctx, fd_sysvar_epoch_rewards_t *
     FD_LOG_ERR(("fd_sysvar_epoch_rewards_encode failed"));
   }
 
-  fd_sysvar_set( slot_ctx, fd_sysvar_owner_id.key, &fd_sysvar_epoch_rewards_id, enc, sz, slot_ctx->slot_bank.slot, acc_lamports );
+  fd_sysvar_set( slot_ctx, fd_sysvar_owner_id.key, &fd_sysvar_epoch_rewards_id, enc, sz, slot_ctx->slot_bank.slot, 0UL );
 }
 
-
 fd_sysvar_epoch_rewards_t *
 fd_sysvar_epoch_rewards_read(
     fd_sysvar_epoch_rewards_t * result,
-    fd_exec_slot_ctx_t  * slot_ctx,
-    fd_acc_lamports_t * acc_lamports
+    fd_exec_slot_ctx_t  * slot_ctx
 ) {
   FD_BORROWED_ACCOUNT_DECL(acc);
   int err = fd_acc_mgr_view( slot_ctx->acc_mgr, slot_ctx->funk_txn, &fd_sysvar_epoch_rewards_id, acc );
-  if( FD_UNLIKELY( err != FD_ACC_MGR_SUCCESS ) )
+  if( FD_UNLIKELY( err != FD_ACC_MGR_SUCCESS ) ) {
     return NULL;
+  }
 
   fd_bincode_decode_ctx_t decode =
     { .data    = acc->const_data,
@@ -62,56 +40,70 @@ fd_sysvar_epoch_rewards_read(
   if( FD_UNLIKELY( fd_sysvar_epoch_rewards_decode( result, &decode )!=FD_BINCODE_SUCCESS ) )
     return NULL;
 
-  if( acc_lamports )
-    *acc_lamports = acc->const_meta->info.lamports;
-
   return result;
 }
 
-/* Update EpochRewards sysvar with distributed rewards */
 void
-fd_sysvar_epoch_rewards_update(
+fd_sysvar_epoch_rewards_distribute(
     fd_exec_slot_ctx_t * slot_ctx,
     ulong distributed
 ) {
-    fd_sysvar_epoch_rewards_t result;
-    fd_acc_lamports_t acc_lamports = 0UL;
-    fd_sysvar_epoch_rewards_read( &result, slot_ctx, &acc_lamports );
-    FD_TEST( acc_lamports != 0 );
+    FD_TEST( FD_FEATURE_ACTIVE( slot_ctx, enable_partitioned_epoch_reward ) );
+
+    fd_sysvar_epoch_rewards_t epoch_rewards[1];
+    if ( FD_UNLIKELY( fd_sysvar_epoch_rewards_read( epoch_rewards, slot_ctx ) == NULL ) ) {
+      FD_LOG_ERR(( "failed to read sysvar epoch rewards" ));
+    }
+    FD_TEST( epoch_rewards->active );
 
-    FD_TEST( result.epoch_rewards.distributed_rewards + distributed <= result.epoch_rewards.total_rewards );
-    result.epoch_rewards.distributed_rewards += distributed;
+    FD_TEST( fd_ulong_sat_add( epoch_rewards->distributed_rewards, distributed ) <= epoch_rewards->total_rewards );
 
-    acc_lamports -= distributed;
+    epoch_rewards->distributed_rewards += distributed;
 
-    write_epoch_rewards( slot_ctx, &result, acc_lamports);
+    write_epoch_rewards( slot_ctx, epoch_rewards );
 }
 
-/* Create EpochRewards syavar with calculated rewards */
+void
+fd_sysvar_epoch_rewards_set_inactive(
+  fd_exec_slot_ctx_t * slot_ctx
+) {
+    fd_sysvar_epoch_rewards_t epoch_rewards[1];
+    if ( FD_UNLIKELY( fd_sysvar_epoch_rewards_read( epoch_rewards, slot_ctx ) == NULL ) ) {
+      FD_LOG_ERR(( "failed to read sysvar epoch rewards" ));
+    }
+    FD_TEST( epoch_rewards->distributed_rewards == epoch_rewards->total_rewards );
+
+    epoch_rewards->active = 0;
+
+    write_epoch_rewards( slot_ctx, epoch_rewards );
+}
+
+/* Create EpochRewards syavar with calculated rewards
+
+   https://github.com/anza-xyz/agave/blob/cbc8320d35358da14d79ebcada4dfb6756ffac79/runtime/src/bank/partitioned_epoch_rewards/sysvar.rs#L25 */
 void
 fd_sysvar_epoch_rewards_init(
     fd_exec_slot_ctx_t * slot_ctx,
     ulong total_rewards,
     ulong distributed_rewards,
-    ulong distribution_complete_block_height
+    ulong distribution_starting_block_height,
+    ulong num_partitions,
+    uint128 total_points,
+    fd_hash_t * last_blockhash
 ) {
+    FD_TEST( FD_FEATURE_ACTIVE( slot_ctx, enable_partitioned_epoch_reward ) );
     FD_TEST( total_rewards >= distributed_rewards );
 
     fd_sysvar_epoch_rewards_t epoch_rewards = {
-        .epoch_rewards= {
-//          .distribution_starting_block_height = distribution_starting_block_height,
-          .distribution_starting_block_height = distribution_complete_block_height,
-//          .num_partitions = num_partitions,
-          .num_partitions = 0,
-//          .parent_blockhash = parent_blockhash,
-//          .total_points = total_points,
-          .total_points = 0,
-          .total_rewards = total_rewards,
-          .distributed_rewards = distributed_rewards,
-          .active = true
-        }
+      .distribution_starting_block_height = distribution_starting_block_height,
+      .num_partitions = num_partitions,
+      .total_points = total_points,
+      .total_rewards = total_rewards,
+      .distributed_rewards = distributed_rewards,
+      .active = 1
     };
-    // set the account lamports to the undistributed rewards
-    fd_acc_lamports_t undistributed_rewards = total_rewards - distributed_rewards;
-    write_epoch_rewards( slot_ctx, &epoch_rewards, undistributed_rewards);
+
+    fd_memcpy( &epoch_rewards.parent_blockhash, last_blockhash, FD_HASH_FOOTPRINT );
+
+    write_epoch_rewards( slot_ctx, &epoch_rewards );
 }