Merge

CHANGELOG.md

@@ -12,7 +12,7 @@
 * State sync is disabled by default [#8730](https://github.com/near/nearcore/pull/8730)
 * Node can restart if State Sync gets interrupted. [#8732](https://github.com/near/nearcore/pull/8732)
 * Merged two `neard view-state` commands: `apply-state-parts` and `dump-state-parts` into a single `state-parts` command. [#8739](https://github.com/near/nearcore/pull/8739)
-* Node can sync State from S3. [#XXXX](https://github.com/near/nearcore/pull/XXXX)
+* Node can sync State from S3. [#8789](https://github.com/near/nearcore/pull/8789)
 
 ## 1.32.0
 

chain/chain/src/test_utils/kv_runtime.rs

@@ -4,6 +4,7 @@ use std::sync::{Arc, RwLock, Weak};
 
 use borsh::{BorshDeserialize, BorshSerialize};
 
+use near_epoch_manager::shard_tracker::{ShardTracker, TrackedConfig};
 use near_epoch_manager::{EpochManagerAdapter, RngSeed};
 use near_primitives::sandbox::state_patch::SandboxStatePatch;
 use near_primitives::state_part::PartId;
@@ -858,6 +859,45 @@ impl EpochManagerAdapter for KeyValueRuntime {
             Ok(())
         }
     }
+
+    fn cares_about_shard_from_prev_block(
+        &self,
+        parent_hash: &CryptoHash,
+        account_id: &AccountId,
+        shard_id: ShardId,
+    ) -> Result<bool, EpochError> {
+        // This `unwrap` here tests that in all code paths we check that the epoch exists before
+        //    we check if we care about a shard. Please do not remove the unwrap, fix the logic of
+        //    the calling function.
+        let epoch_valset = self.get_epoch_and_valset(*parent_hash).unwrap();
+        let chunk_producers = self.get_chunk_producers(epoch_valset.1, shard_id);
+        for validator in chunk_producers {
+            if validator.account_id() == account_id {
+                return Ok(true);
+            }
+        }
+        Ok(false)
+    }
+
+    fn cares_about_shard_next_epoch_from_prev_block(
+        &self,
+        parent_hash: &CryptoHash,
+        account_id: &AccountId,
+        shard_id: ShardId,
+    ) -> Result<bool, EpochError> {
+        // This `unwrap` here tests that in all code paths we check that the epoch exists before
+        //    we check if we care about a shard. Please do not remove the unwrap, fix the logic of
+        //    the calling function.
+        let epoch_valset = self.get_epoch_and_valset(*parent_hash).unwrap();
+        let chunk_producers = self
+            .get_chunk_producers((epoch_valset.1 + 1) % self.validators_by_valset.len(), shard_id);
+        for validator in chunk_producers {
+            if validator.account_id() == account_id {
+                return Ok(true);
+            }
+        }
+        Ok(false)
+    }
 }
 
 impl RuntimeAdapter for KeyValueRuntime {
@@ -936,19 +976,12 @@ impl RuntimeAdapter for KeyValueRuntime {
         if self.tracks_all_shards {
             return true;
         }
-        // This `unwrap` here tests that in all code paths we check that the epoch exists before
-        //    we check if we care about a shard. Please do not remove the unwrap, fix the logic of
-        //    the calling function.
-        let epoch_valset = self.get_epoch_and_valset(*parent_hash).unwrap();
-        let chunk_producers = self.get_chunk_producers(epoch_valset.1, shard_id);
         if let Some(account_id) = account_id {
-            for validator in chunk_producers {
-                if validator.account_id() == account_id {
-                    return true;
-                }
-            }
+            self.cares_about_shard_from_prev_block(parent_hash, account_id, shard_id)
+                .unwrap_or(false)
+        } else {
+            false
         }
-        false
     }
 
     fn will_care_about_shard(
@@ -961,20 +994,12 @@ impl RuntimeAdapter for KeyValueRuntime {
         if self.tracks_all_shards {
             return true;
         }
-        // This `unwrap` here tests that in all code paths we check that the epoch exists before
-        //    we check if we care about a shard. Please do not remove the unwrap, fix the logic of
-        //    the calling function.
-        let epoch_valset = self.get_epoch_and_valset(*parent_hash).unwrap();
-        let chunk_producers = self
-            .get_chunk_producers((epoch_valset.1 + 1) % self.validators_by_valset.len(), shard_id);
         if let Some(account_id) = account_id {
-            for validator in chunk_producers {
-                if validator.account_id() == account_id {
-                    return true;
-                }
-            }
+            self.cares_about_shard_next_epoch_from_prev_block(parent_hash, account_id, shard_id)
+                .unwrap_or(false)
+        } else {
+            false
         }
-        false
     }
 
     fn validate_tx(
@@ -1433,4 +1458,12 @@ impl RuntimeWithEpochManagerAdapter for KeyValueRuntime {
     fn epoch_manager_adapter_arc(&self) -> Arc<dyn EpochManagerAdapter> {
         self.myself.upgrade().unwrap()
     }
+    fn shard_tracker(&self) -> ShardTracker {
+        let config = if self.tracks_all_shards {
+            TrackedConfig::AllShards
+        } else {
+            TrackedConfig::new_empty()
+        };
+        ShardTracker::new(config, self.epoch_manager_adapter_arc())
+    }
 }

chain/chain/src/types.rs

@@ -4,6 +4,7 @@ use std::sync::Arc;
 use borsh::{BorshDeserialize, BorshSerialize};
 use chrono::DateTime;
 use chrono::Utc;
+use near_epoch_manager::shard_tracker::ShardTracker;
 use near_primitives::sandbox::state_patch::SandboxStatePatch;
 use num_rational::Rational32;
 
@@ -579,6 +580,7 @@ pub trait RuntimeAdapter: Send + Sync {
 pub trait RuntimeWithEpochManagerAdapter: RuntimeAdapter + EpochManagerAdapter {
     fn epoch_manager_adapter(&self) -> &dyn EpochManagerAdapter;
     fn epoch_manager_adapter_arc(&self) -> Arc<dyn EpochManagerAdapter>;
+    fn shard_tracker(&self) -> ShardTracker;
 }
 
 /// The last known / checked height and time when we have processed it.

chain/client/src/client.rs

@@ -41,6 +41,7 @@ use near_primitives::block::{Approval, ApprovalInner, ApprovalMessage, Block, Bl
 use near_primitives::block_header::ApprovalType;
 use near_primitives::challenge::{Challenge, ChallengeBody};
 use near_primitives::epoch_manager::RngSeed;
+use near_primitives::errors::EpochError;
 use near_primitives::hash::CryptoHash;
 use near_primitives::merkle::{merklize, MerklePath, PartialMerkleTree};
 use near_primitives::network::PeerId;
@@ -1758,7 +1759,7 @@ impl Client {
         let next_block_epoch_id =
             match self.runtime_adapter.get_epoch_id_from_prev_block(&parent_hash) {
                 Err(e) => {
-                    self.handle_process_approval_error(approval, approval_type, true, e);
+                    self.handle_process_approval_error(approval, approval_type, true, e.into());
                     return;
                 }
                 Ok(next_epoch_id) => next_epoch_id,
@@ -1779,7 +1780,7 @@ impl Client {
                 account_id,
             ) {
                 Ok(_) => next_block_epoch_id.clone(),
-                Err(near_chain::Error::NotAValidator) => {
+                Err(EpochError::NotAValidator(_, _)) => {
                     match self.runtime_adapter.get_next_epoch_id_from_prev_block(&parent_hash) {
                         Ok(next_block_next_epoch_id) => next_block_next_epoch_id,
                         Err(_) => return,

chain/client/src/client_actor.rs

@@ -33,6 +33,7 @@ use near_chain::{
     ChainGenesis, DoneApplyChunkCallback, Provenance, RuntimeWithEpochManagerAdapter,
 };
 use near_chain_configs::{ClientConfig, LogSummaryStyle};
+use near_chain_primitives::error::EpochErrorResultToChainError;
 use near_chunks::adapter::ShardsManagerRequestFromClient;
 use near_chunks::client::ShardsManagerResponse;
 use near_chunks::logic::cares_about_shard_this_or_next_epoch;
@@ -652,7 +653,8 @@ impl Handler<WithSpanContext<Status>> for ClientActor {
         let validators: Vec<ValidatorInfo> = self
             .client
             .runtime_adapter
-            .get_epoch_block_producers_ordered(&head.epoch_id, &head.last_block_hash)?
+            .get_epoch_block_producers_ordered(&head.epoch_id, &head.last_block_hash)
+            .into_chain_error()?
             .into_iter()
             .map(|(validator_stake, is_slashed)| ValidatorInfo {
                 account_id: validator_stake.take_account_id(),
@@ -663,8 +665,11 @@ impl Handler<WithSpanContext<Status>> for ClientActor {
         let epoch_start_height =
             self.client.runtime_adapter.get_epoch_start_height(&head.last_block_hash).ok();
 
-        let protocol_version =
-            self.client.runtime_adapter.get_epoch_protocol_version(&head.epoch_id)?;
+        let protocol_version = self
+            .client
+            .runtime_adapter
+            .get_epoch_protocol_version(&head.epoch_id)
+            .into_chain_error()?;
 
         let node_public_key = self.node_id.public_key().clone();
         let (validator_account_id, validator_public_key) = match &self.client.validator_signer {

nearcore/src/config.rs

@@ -313,8 +313,8 @@ pub struct Config {
     /// save_trie_changes = !archive
     /// save_trie_changes should be set to true iff
     /// - archive if false - non-archival nodes need trie changes to perform garbage collection
-    /// - archive is true, cold_store is configured and migration to split_storage is finished - node
-    /// working in split storage mode needs trie changes in order to do garbage collection on hot.
+    /// - archive is true and cold_store is configured - node working in split storage mode
+    /// needs trie changes in order to do garbage collection on hot and populate cold State column.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub save_trie_changes: Option<bool>,
     pub log_summary_style: LogSummaryStyle,

nearcore/src/lib.rs

@@ -1,7 +1,6 @@
 use crate::cold_storage::spawn_cold_store_loop;
 pub use crate::config::{init_configs, load_config, load_test_config, NearConfig, NEAR_BASE};
 pub use crate::runtime::NightshadeRuntime;
-pub use crate::shard_tracker::TrackedConfig;
 use crate::state_sync::{spawn_state_sync_dump, StateSyncDumpHandle};
 use actix::{Actor, Addr};
 use actix_rt::ArbiterHandle;
@@ -33,7 +32,6 @@ pub mod dyn_config;
 mod metrics;
 pub mod migrations;
 mod runtime;
-mod shard_tracker;
 mod state_sync;
 
 pub fn get_default_home() -> PathBuf {

nearcore/src/runtime/mod.rs

@@ -1,6 +1,5 @@
 use crate::metrics;
 use crate::migrations::load_migration_data;
-use crate::shard_tracker::{ShardTracker, TrackedConfig};
 use crate::NearConfig;
 use borsh::ser::BorshSerialize;
 use borsh::BorshDeserialize;
@@ -15,6 +14,7 @@ use near_chain_configs::{
 };
 use near_client_primitives::types::StateSplitApplyingStatus;
 use near_crypto::PublicKey;
+use near_epoch_manager::shard_tracker::{ShardTracker, TrackedConfig};
 use near_epoch_manager::{EpochManager, EpochManagerAdapter, EpochManagerHandle};
 use near_o11y::log_assert;
 use near_pool::types::PoolIterator;
@@ -153,7 +153,7 @@ impl NightshadeRuntime {
             EpochManager::new_from_genesis_config(store.clone().into(), &genesis_config)
                 .expect("Failed to start Epoch Manager")
                 .into_handle();
-        let shard_tracker = ShardTracker::new(tracked_config, epoch_manager.clone());
+        let shard_tracker = ShardTracker::new(tracked_config, Arc::new(epoch_manager.clone()));
         Arc::new_cyclic(|myself| NightshadeRuntime {
             genesis_config,
             runtime_config_store,
@@ -1488,6 +1488,10 @@ impl RuntimeWithEpochManagerAdapter for NightshadeRuntime {
     fn epoch_manager_adapter_arc(&self) -> Arc<dyn EpochManagerAdapter> {
         self.myself.upgrade().unwrap()
     }
+
+    fn shard_tracker(&self) -> ShardTracker {
+        self.shard_tracker.clone()
+    }
 }
 
 impl node_runtime::adapter::ViewRuntimeAdapter for NightshadeRuntime {
@@ -1590,6 +1594,7 @@ mod test {
     use std::collections::BTreeSet;
 
     use near_chain::{Chain, ChainGenesis};
+    use near_epoch_manager::shard_tracker::TrackedConfig;
     use near_primitives::test_utils::create_test_signer;
     use near_primitives::types::validator_stake::ValidatorStake;
     use near_store::flat::{FlatStateChanges, FlatStateDelta, FlatStateDeltaMetadata};