Change ShardTracker interface (#4668)

* change ShardTracker interface

* fix csv-to-json

* fix merge

* make shard tracker work when shard layout changes and add test for that

* revert change on client config. Now this PR does not change config format

* fix CI

* comments and CI

* Fix CI

* address comments

* fix AppendOnlyMap

* add comments and fix fmt

* move AppendOnlyMap to a separate file

* create TrackedConfig

chain/client/src/view_client.rs

@@ -1095,16 +1095,30 @@ impl Handler<NetworkViewClientMessages> for ViewClientActor {
             }
             NetworkViewClientMessages::GetChainInfo => match self.chain.head() {
                 Ok(head) => {
-                    let height = self.get_height(&head);
-
-                    NetworkViewClientResponses::ChainInfo {
-                        genesis_id: GenesisId {
-                            chain_id: self.config.chain_id.clone(),
-                            hash: *self.chain.genesis().hash(),
-                        },
-                        height,
-                        tracked_shards: self.config.tracked_shards.clone(),
-                        archival: self.config.archive,
+                    match self.runtime_adapter.num_shards(&head.epoch_id) {
+                        Ok(num_shards) => {
+                            // convert config tracked shards
+                            // runtime will track all shards if config tracked shards is not empty
+                            // https://github.com/near/nearcore/issues/4930
+                            let tracked_shards = if self.config.tracked_shards.is_empty() {
+                                vec![]
+                            } else {
+                                (0..num_shards).collect()
+                            };
+                            NetworkViewClientResponses::ChainInfo {
+                                genesis_id: GenesisId {
+                                    chain_id: self.config.chain_id.clone(),
+                                    hash: *self.chain.genesis().hash(),
+                                },
+                                height: self.get_height(&head),
+                                tracked_shards,
+                                archival: self.config.archive,
+                            }
+                        }
+                        Err(err) => {
+                            error!(target: "view_client", "Cannot retrieve num shards: {}", err);
+                            NetworkViewClientResponses::NoResponse
+                        }
                     }
                 }
                 Err(err) => {

genesis-tools/genesis-populate/src/lib.rs

@@ -26,7 +26,7 @@ use near_primitives::types::{AccountId, Balance, EpochId, ShardId, StateChangeCa
 use near_store::{
     create_store, get_account, set_access_key, set_account, set_code, Store, TrieUpdate,
 };
-use nearcore::{get_store_path, NightshadeRuntime};
+use nearcore::{get_store_path, NightshadeRuntime, TrackedConfig};
 
 fn get_account_id(account_index: u64) -> AccountId {
     AccountId::try_from(format!("near_{}_{}", account_index, account_index)).unwrap()
@@ -67,8 +67,7 @@ impl GenesisBuilder {
             &genesis,
             // Since we are not using runtime as an actor
             // there is no reason to track accounts or shards.
-            vec![],
-            vec![],
+            TrackedConfig::new_empty(),
             None,
             None,
             RuntimeConfigStore::new(Some(&genesis.config.runtime_config)),

integration-tests/src/genesis_helpers.rs

@@ -6,7 +6,6 @@ use near_chain::{Chain, ChainGenesis, DoomslugThresholdMode};
 use near_chain_configs::Genesis;
 use near_primitives::block::{Block, BlockHeader};
 use near_primitives::hash::CryptoHash;
-use near_primitives::runtime::config_store::RuntimeConfigStore;
 use near_store::test_utils::create_test_store;
 use nearcore::NightshadeRuntime;
 
@@ -20,16 +19,7 @@ pub fn genesis_header(genesis: &Genesis) -> BlockHeader {
     let dir = tempdir().unwrap();
     let store = create_test_store();
     let chain_genesis = ChainGenesis::from(genesis);
-    let runtime = Arc::new(NightshadeRuntime::new(
-        dir.path(),
-        store,
-        genesis,
-        vec![],
-        vec![],
-        None,
-        None,
-        RuntimeConfigStore::test(),
-    ));
+    let runtime = Arc::new(NightshadeRuntime::test(dir.path(), store, genesis));
     let chain = Chain::new(runtime, &chain_genesis, DoomslugThresholdMode::TwoThirds).unwrap();
     chain.genesis().clone()
 }
@@ -39,16 +29,7 @@ pub fn genesis_block(genesis: &Genesis) -> Block {
     let dir = tempdir().unwrap();
     let store = create_test_store();
     let chain_genesis = ChainGenesis::from(genesis);
-    let runtime = Arc::new(NightshadeRuntime::new(
-        dir.path(),
-        store,
-        genesis,
-        vec![],
-        vec![],
-        None,
-        None,
-        RuntimeConfigStore::test(),
-    ));
+    let runtime = Arc::new(NightshadeRuntime::test(dir.path(), store, genesis));
     let mut chain = Chain::new(runtime, &chain_genesis, DoomslugThresholdMode::TwoThirds).unwrap();
     chain.get_block(&chain.genesis().hash().clone()).unwrap().clone()
 }

integration-tests/tests/client/challenges.rs

@@ -23,7 +23,6 @@ use near_primitives::hash::CryptoHash;
 use near_primitives::merkle::{merklize, MerklePath, PartialMerkleTree};
 use near_primitives::num_rational::Rational;
 use near_primitives::receipt::Receipt;
-use near_primitives::runtime::config_store::RuntimeConfigStore;
 use near_primitives::serialize::BaseDecode;
 use near_primitives::sharding::{EncodedShardChunk, ReedSolomonWrapper};
 use near_primitives::transaction::SignedTransaction;
@@ -272,15 +271,10 @@ fn test_verify_chunk_invalid_state_challenge() {
     let genesis = Genesis::test(vec!["test0".parse().unwrap(), "test1".parse().unwrap()], 1);
     let transaction_validity_period = genesis.config.transaction_validity_period;
     let mut env = TestEnv::builder(ChainGenesis::test())
-        .runtime_adapters(vec![Arc::new(nearcore::NightshadeRuntime::new(
+        .runtime_adapters(vec![Arc::new(nearcore::NightshadeRuntime::test(
             Path::new("."),
             store1,
             &genesis,
-            vec![],
-            vec![],
-            None,
-            None,
-            RuntimeConfigStore::test(),
         ))])
         .build();
     let signer = InMemorySigner::from_seed("test0".parse().unwrap(), KeyType::ED25519, "test0");
@@ -575,15 +569,10 @@ fn test_fishermen_challenge() {
     );
     genesis.config.epoch_length = 5;
     let create_runtime = || -> Arc<NightshadeRuntime> {
-        Arc::new(nearcore::NightshadeRuntime::new(
+        Arc::new(nearcore::NightshadeRuntime::test(
             Path::new("."),
             create_test_store(),
             &genesis.clone(),
-            vec![],
-            vec![],
-            None,
-            None,
-            RuntimeConfigStore::test(),
         ))
     };
     let runtime1 = create_runtime();
@@ -641,25 +630,15 @@ fn test_challenge_in_different_epoch() {
     genesis.config.epoch_length = 3;
     //    genesis.config.validator_kickout_threshold = 10;
     let network_adapter = Arc::new(MockNetworkAdapter::default());
-    let runtime1 = Arc::new(nearcore::NightshadeRuntime::new(
+    let runtime1 = Arc::new(nearcore::NightshadeRuntime::test(
         Path::new("."),
         create_test_store(),
         &genesis.clone(),
-        vec![],
-        vec![],
-        None,
-        None,
-        RuntimeConfigStore::test(),
     ));
-    let runtime2 = Arc::new(nearcore::NightshadeRuntime::new(
+    let runtime2 = Arc::new(nearcore::NightshadeRuntime::test(
         Path::new("."),
         create_test_store(),
         &genesis.clone(),
-        vec![],
-        vec![],
-        None,
-        None,
-        RuntimeConfigStore::test(),
     ));
     let mut chain_genesis = ChainGenesis::test();
     chain_genesis.epoch_length = 3;

integration-tests/tests/client/process_blocks.rs

@@ -41,7 +41,6 @@ use near_primitives::errors::TxExecutionError;
 use near_primitives::hash::{hash, CryptoHash};
 use near_primitives::merkle::verify_hash;
 use near_primitives::receipt::DelayedReceiptIndices;
-use near_primitives::runtime::config_store::RuntimeConfigStore;
 use near_primitives::shard_layout::ShardUId;
 #[cfg(not(feature = "protocol_feature_block_header_v3"))]
 use near_primitives::sharding::ShardChunkHeaderV2;
@@ -86,15 +85,10 @@ pub fn set_block_protocol_version(
 pub fn create_nightshade_runtimes(genesis: &Genesis, n: usize) -> Vec<Arc<dyn RuntimeAdapter>> {
     (0..n)
         .map(|_| {
-            Arc::new(nearcore::NightshadeRuntime::new(
+            Arc::new(nearcore::NightshadeRuntime::test(
                 Path::new("."),
                 create_test_store(),
                 genesis,
-                vec![],
-                vec![],
-                None,
-                None,
-                RuntimeConfigStore::test(),
             )) as Arc<dyn RuntimeAdapter>
         })
         .collect()
@@ -1913,16 +1907,8 @@ fn test_invalid_block_root() {
 fn test_incorrect_validator_key_produce_block() {
     let genesis = Genesis::test(vec!["test0".parse().unwrap(), "test1".parse().unwrap()], 2);
     let chain_genesis = ChainGenesis::from(&genesis);
-    let runtime_adapter: Arc<dyn RuntimeAdapter> = Arc::new(nearcore::NightshadeRuntime::new(
-        Path::new("."),
-        create_test_store(),
-        &genesis,
-        vec![],
-        vec![],
-        None,
-        None,
-        RuntimeConfigStore::test(),
-    ));
+    let runtime_adapter: Arc<dyn RuntimeAdapter> =
+        Arc::new(nearcore::NightshadeRuntime::test(Path::new("."), create_test_store(), &genesis));
     let signer = Arc::new(InMemoryValidatorSigner::from_seed(
         "test0".parse().unwrap(),
         KeyType::ED25519,
@@ -3383,16 +3369,8 @@ mod protocol_feature_restore_receipts_after_fix_tests {
         genesis.config.epoch_length = EPOCH_LENGTH;
         genesis.config.protocol_version = protocol_version;
         let chain_genesis = ChainGenesis::from(&genesis);
-        let runtime = nearcore::NightshadeRuntime::new(
-            Path::new("."),
-            create_test_store(),
-            &genesis,
-            vec![],
-            vec![],
-            None,
-            None,
-            RuntimeConfigStore::test(),
-        );
+        let runtime =
+            nearcore::NightshadeRuntime::test(Path::new("."), create_test_store(), &genesis);
         // TODO #4305: get directly from NightshadeRuntime
         let migration_data = load_migration_data(&genesis.config.chain_id);
 
@@ -3686,16 +3664,8 @@ mod contract_precompilation_tests {
         let runtime_adapters = stores
             .iter()
             .map(|store| {
-                Arc::new(nearcore::NightshadeRuntime::new(
-                    Path::new("."),
-                    store.clone(),
-                    &genesis,
-                    vec![],
-                    vec![],
-                    None,
-                    None,
-                    RuntimeConfigStore::test(),
-                )) as Arc<dyn RuntimeAdapter>
+                Arc::new(nearcore::NightshadeRuntime::test(Path::new("."), store.clone(), &genesis))
+                    as Arc<dyn RuntimeAdapter>
             })
             .collect();
 
@@ -3791,16 +3761,8 @@ mod contract_precompilation_tests {
         let runtime_adapters = stores
             .iter()
             .map(|store| {
-                Arc::new(nearcore::NightshadeRuntime::new(
-                    Path::new("."),
-                    store.clone(),
-                    &genesis,
-                    vec![],
-                    vec![],
-                    None,
-                    None,
-                    RuntimeConfigStore::test(),
-                )) as Arc<dyn RuntimeAdapter>
+                Arc::new(nearcore::NightshadeRuntime::test(Path::new("."), store.clone(), &genesis))
+                    as Arc<dyn RuntimeAdapter>
             })
             .collect();
 
@@ -3874,16 +3836,8 @@ mod contract_precompilation_tests {
         let runtime_adapters = stores
             .iter()
             .map(|store| {
-                Arc::new(nearcore::NightshadeRuntime::new(
-                    Path::new("."),
-                    store.clone(),
-                    &genesis,
-                    vec![],
-                    vec![],
-                    None,
-                    None,
-                    RuntimeConfigStore::test(),
-                )) as Arc<dyn RuntimeAdapter>
+                Arc::new(nearcore::NightshadeRuntime::test(Path::new("."), store.clone(), &genesis))
+                    as Arc<dyn RuntimeAdapter>
             })
             .collect();
 

integration-tests/tests/client/runtimes.rs

@@ -19,7 +19,6 @@ use near_primitives::block::{Approval, ApprovalInner};
 use near_primitives::block_header::ApprovalType;
 use near_primitives::hash::hash;
 use near_primitives::network::PeerId;
-use near_primitives::runtime::config_store::RuntimeConfigStore;
 #[cfg(feature = "protocol_feature_block_header_v3")]
 use near_primitives::sharding::ShardChunkHeaderInner;
 use near_primitives::sharding::{PartialEncodedChunk, ShardChunkHeader};
@@ -32,15 +31,10 @@ use nearcore::config::GenesisExt;
 pub fn create_nightshade_runtimes(genesis: &Genesis, n: usize) -> Vec<Arc<dyn RuntimeAdapter>> {
     (0..n)
         .map(|_| {
-            Arc::new(nearcore::NightshadeRuntime::new(
+            Arc::new(nearcore::NightshadeRuntime::test(
                 Path::new("."),
                 create_test_store(),
                 &genesis,
-                vec![],
-                vec![],
-                None,
-                None,
-                RuntimeConfigStore::test(),
             )) as Arc<dyn RuntimeAdapter>
         })
         .collect()

integration-tests/tests/client/sandbox.rs

@@ -6,7 +6,6 @@ use near_chain_configs::Genesis;
 use near_client::test_utils::TestEnv;
 use near_crypto::{InMemorySigner, KeyType};
 use near_primitives::account::Account;
-use near_primitives::runtime::config_store::RuntimeConfigStore;
 use near_primitives::serialize::{from_base64, to_base64};
 use near_primitives::state_record::StateRecord;
 use near_primitives::transaction::{
@@ -21,15 +20,10 @@ fn test_setup() -> (TestEnv, InMemorySigner) {
     let mut genesis = Genesis::test(vec!["test0".parse().unwrap(), "test1".parse().unwrap()], 1);
     genesis.config.epoch_length = epoch_length;
     let mut env = TestEnv::builder(ChainGenesis::test())
-        .runtime_adapters(vec![Arc::new(nearcore::NightshadeRuntime::new(
+        .runtime_adapters(vec![Arc::new(nearcore::NightshadeRuntime::test(
             Path::new("."),
             create_test_store(),
             &genesis,
-            vec![],
-            vec![],
-            None,
-            None,
-            RuntimeConfigStore::test(),
         )) as Arc<dyn RuntimeAdapter>])
         .build();
     let signer = InMemorySigner::from_seed("test0".parse().unwrap(), KeyType::ED25519, "test0");

nearcore/src/append_only_map.rs

@@ -0,0 +1,26 @@
+use std::collections::HashMap;
+use std::hash::Hash;
+use std::sync::{Arc, RwLock};
+
+const POISONED_LOCK_ERR: &str = "The lock was poisoned.";
+
+pub struct AppendOnlyMap<K, V> {
+    map: RwLock<HashMap<K, Arc<V>>>,
+}
+
+impl<K, V> AppendOnlyMap<K, V>
+where
+    K: Eq + Hash + Clone,
+{
+    pub fn new() -> Self {
+        Self { map: RwLock::new(HashMap::new()) }
+    }
+
+    pub fn get_or_insert<F: FnOnce() -> V>(&self, key: &K, value: F) -> Arc<V> {
+        let mut map = self.map.write().expect(POISONED_LOCK_ERR);
+        if !map.contains_key(key) {
+            map.insert(key.clone(), Arc::new(value()));
+        }
+        map.get(key).unwrap().clone()
+    }
+}