Merge branch 'master' into param_estim

Cargo.toml

@@ -70,8 +70,8 @@ neard = { path = "./neard"}
 
 testlib = { path = "./test-utils/testlib" }
 
-[replace]
-"ethereum-types:0.10.0" = { path = "./patches/ethereum-types-0.10.0-to-0.11.0" }
+[patch.crates-io]
+ethereum-types = { path = "./patches/ethereum-types-0.10.0-to-0.11.0" }
 
 [profile.release]
 lto = true        # Enable full link-time optimization.

README.md

@@ -2,7 +2,7 @@
 <br />
 
 <p align="center">
-<img src="docs/logo.svg" width="240">
+<img src="docs/images/logo.svg" width="240">
 </p>
 
 <br />

chain/client/src/view_client.rs

@@ -630,7 +630,17 @@ impl Handler<GetValidatorInfo> for ViewClientActor {
     #[perf]
     fn handle(&mut self, msg: GetValidatorInfo, _: &mut Self::Context) -> Self::Result {
         let epoch_identifier = match msg.epoch_reference {
-            EpochReference::EpochId(id) => ValidatorInfoIdentifier::EpochId(id),
+            EpochReference::EpochId(id) => {
+                // By `EpochId` we can get only cached epochs.
+                // Request for not finished epoch by `EpochId` will return an error because epoch has not been cached yet
+                // If the requested one is current ongoing we need to handle it like `Latest`
+                let tip = self.chain.header_head()?;
+                if tip.epoch_id == id {
+                    ValidatorInfoIdentifier::BlockHash(tip.last_block_hash)
+                } else {
+                    ValidatorInfoIdentifier::EpochId(id)
+                }
+            }
             EpochReference::BlockId(block_id) => {
                 let block_header = match block_id {
                     BlockId::Hash(h) => self.chain.get_block_header(&h)?.clone(),

chain/jsonrpc/Cargo.toml

@@ -6,8 +6,9 @@ edition = "2018"
 
 [dependencies]
 actix = "0.11.0-beta.2"
-actix-web = "4.0.0-beta.3"
-actix-cors = { git = "https://github.com/near/actix-extras.git", branch="actix-web-4-beta.3" }
+awc = "3.0.0-beta.5"
+actix-web = "4.0.0-beta.6"
+actix-cors = { git = "https://github.com/near/actix-extras.git", branch="actix-web-4-beta.6" }
 easy-ext = "0.2"
 tokio = { version = "1.1", features = ["full"] }
 futures = "0.3"

chain/jsonrpc/client/Cargo.toml

@@ -5,7 +5,8 @@ authors = ["Near Inc <hello@nearprotocol.com>"]
 edition = "2018"
 
 [dependencies]
-actix-web = "4.0.0-beta.1"
+awc = "3.0.0-beta.5"
+actix-web = "4.0.0-beta.6"
 futures = "0.3"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"

chain/jsonrpc/client/src/lib.rs

@@ -1,6 +1,6 @@
 use std::time::Duration;
 
-use actix_web::client::{Client, Connector};
+use awc::{Client, Connector};
 use futures::{future, future::LocalBoxFuture, FutureExt, TryFutureExt};
 use serde::Deserialize;
 use serde::Serialize;
@@ -260,8 +260,7 @@ fn create_client() -> Client {
         .connector(
             Connector::new()
                 .conn_lifetime(Duration::from_secs(u64::max_value()))
-                .conn_keep_alive(Duration::from_secs(30))
-                .finish(),
+                .conn_keep_alive(Duration::from_secs(30)),
         )
         .finish()
 }

chain/jsonrpc/tests/test_utils/mod.rs

@@ -71,7 +71,7 @@ type RpcRequest<T> = LocalBoxFuture<'static, Result<T, near_jsonrpc_primitives::
 
 /// Prepare a `RPCRequest` with a given client, server address, method and parameters.
 pub fn call_method<R>(
-    client: &actix_web::client::Client,
+    client: &awc::Client,
     server_addr: &str,
     method: &str,
     params: serde_json::Value,

chain/rosetta-rpc/Cargo.toml

@@ -12,17 +12,18 @@ hex = "0.4"
 lazy_static = "1.4"
 strum = { version = "0.20", features = ["derive"] }
 
+awc = "3.0.0-beta.5"
 actix = "0.11.0-beta.2"
-actix-web = "4.0.0-beta.3"
-actix-cors = { git = "https://github.com/near/actix-extras.git", branch="actix-web-4-beta.3" }
+actix-web = "4.0.0-beta.6"
+actix-cors = { git = "https://github.com/near/actix-extras.git", branch="actix-web-4-beta.6" }
 futures = "0.3.5"
 tokio = { version = "1.1", features = ["full"] }
 
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 validator = "0.12"
 
-paperclip = { git = "https://github.com/frol/paperclip", branch = "actix-web-4-beta.3", features = ["actix"] }
+paperclip = { git = "https://github.com/near/paperclip", branch = "actix-web-4-beta.6", features = ["actix"] }
 
 near-primitives = { path = "../../core/primitives" }
 near-crypto = { path = "../../core/crypto" }

chain/rosetta-rpc/src/models.rs

@@ -543,10 +543,9 @@ where
 }
 
 impl actix_web::ResponseError for Error {
-    fn error_response(&self) -> actix_web::HttpResponse {
+    fn error_response(&self) -> actix_web::BaseHttpResponse<actix_web::body::Body> {
         let data = paperclip::actix::web::Json(self);
-        actix_web::HttpResponse::build(actix_web::http::StatusCode::INTERNAL_SERVER_ERROR)
-            .json(data)
+        actix_web::HttpResponse::InternalServerError().json(data).into()
     }
 }
 

chain/telemetry/Cargo.toml

@@ -6,7 +6,8 @@ edition = "2018"
 
 [dependencies]
 openssl = { version = "0.10", features = ["vendored"] }
-actix-web = { version = "4.0.0-beta.1", features = [ "openssl" ] }
+awc = "3.0.0-beta.5"
+actix-web = { version = "4.0.0-beta.6", features = [ "openssl" ] }
 futures = "0.3"
 actix = "0.11.0-beta.2"
 serde = { version = "1", features = [ "derive" ] }

chain/telemetry/src/lib.rs

@@ -1,7 +1,7 @@
 use std::time::Duration;
 
 use actix::{Actor, Addr, Context, Handler, Message};
-use actix_web::client::{Client, Connector};
+use awc::{Client, Connector};
 use futures::FutureExt;
 use near_performance_metrics_macros::perf;
 use serde::{Deserialize, Serialize};
@@ -49,8 +49,7 @@ impl TelemetryActor {
             .connector(
                 Connector::new()
                     .conn_lifetime(Duration::from_secs(u64::max_value()))
-                    .conn_keep_alive(Duration::from_secs(30))
-                    .finish(),
+                    .conn_keep_alive(Duration::from_secs(30)),
             )
             .finish();
         Self { config, client }

core/chain-configs/src/genesis_config.rs

@@ -4,13 +4,15 @@
 //! contains `RuntimeConfig`, but we keep it here for now until we figure
 //! out the better place.
 use std::fs::File;
-use std::io::BufReader;
+use std::io::{BufReader, Read};
 use std::marker::PhantomData;
-use std::path::Path;
+use std::path::{Path, PathBuf};
+use std::{fmt, io};
 
 use chrono::{DateTime, Utc};
 use num_rational::Rational;
-use serde::{Deserialize, Serialize};
+use serde::de::{self, DeserializeSeed, IgnoredAny, MapAccess, SeqAccess, Visitor};
+use serde::{Deserialize, Deserializer, Serialize};
 use serde_json::Serializer;
 use smart_default::SmartDefault;
 
@@ -180,6 +182,12 @@ pub struct Genesis {
     #[serde(flatten)]
     pub config: GenesisConfig,
     pub records: GenesisRecords,
+    /// Genesis object may not contain records.
+    /// In this case records can be found in records_file.
+    /// The idea is that all records consume too much memory,
+    /// so they should be processed in streaming fashion with for_each_record.
+    #[serde(skip)]
+    pub records_file: PathBuf,
     /// Using zero-size PhantomData is a Rust pattern preventing a structure being constructed
     /// without calling `new` method, which has some initialization routine.
     #[serde(skip)]
@@ -266,6 +274,74 @@ impl GenesisRecords {
     }
 }
 
+/// Visitor for records.
+/// Reads records one by one and passes them to sink.
+/// If full genesis file is passed, reads records from "records" field and
+/// IGNORES OTHER FIELDS.
+struct RecordsProcessor<F> {
+    sink: F,
+}
+
+impl<'de, F: FnMut(StateRecord)> Visitor<'de> for RecordsProcessor<&'_ mut F> {
+    type Value = ();
+
+    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+        formatter.write_str(
+            "either:\
+        1. array of StateRecord\
+        2. map with records field which is array of StateRecord",
+        )
+    }
+
+    fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
+    where
+        A: SeqAccess<'de>,
+    {
+        while let Some(record) = seq.next_element::<StateRecord>()? {
+            (self.sink)(record)
+        }
+        Ok(())
+    }
+
+    fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error>
+    where
+        A: MapAccess<'de>,
+    {
+        while let Some(key) = map.next_key::<String>()? {
+            match key.as_str() {
+                "records" => {
+                    map.next_value_seed(self)?;
+                    return Ok(());
+                }
+                _ => {
+                    map.next_value::<IgnoredAny>()?;
+                }
+            }
+        }
+        Err(de::Error::custom("missing field: records"))
+    }
+}
+
+impl<'de, F: FnMut(StateRecord)> DeserializeSeed<'de> for RecordsProcessor<&'_ mut F> {
+    type Value = ();
+
+    fn deserialize<D>(self, deserializer: D) -> Result<Self::Value, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        deserializer.deserialize_seq(self)
+    }
+}
+
+fn stream_records_from_file(
+    reader: impl Read,
+    mut callback: impl FnMut(StateRecord),
+) -> serde_json::Result<()> {
+    let mut deserializer = serde_json::Deserializer::from_reader(reader);
+    let records_processor = RecordsProcessor { sink: &mut callback };
+    deserializer.deserialize_any(records_processor)
+}
+
 pub struct GenesisJsonHasher {
     digest: sha2::Sha256,
 }
@@ -287,9 +363,9 @@ impl GenesisJsonHasher {
 
     pub fn process_genesis(&mut self, genesis: &Genesis) {
         self.process_config(&genesis.config);
-        for record in genesis.records.as_ref() {
-            self.process_record(record)
-        }
+        genesis.for_each_record(|record: &StateRecord| {
+            self.process_record(record);
+        });
     }
 
     pub fn finalize(self) -> CryptoHash {
@@ -299,11 +375,16 @@ impl GenesisJsonHasher {
 
 impl Genesis {
     pub fn new(config: GenesisConfig, records: GenesisRecords) -> Self {
-        let mut genesis = Self { config, records, phantom: PhantomData };
+        let mut genesis =
+            Self { config, records, records_file: PathBuf::new(), phantom: PhantomData };
         genesis.config.total_supply = get_initial_supply(&genesis.records.as_ref());
         genesis
     }
 
+    pub fn new_with_path(config: GenesisConfig, records_file: PathBuf) -> Self {
+        Self { config, records: GenesisRecords(vec![]), records_file, phantom: PhantomData }
+    }
+
     /// Reads Genesis from a single file.
     pub fn from_file<P: AsRef<Path>>(path: P) -> Self {
         let reader = BufReader::new(File::open(path).expect("Could not open genesis config file."));
@@ -337,6 +418,27 @@ impl Genesis {
         hasher.process_genesis(self);
         hasher.finalize()
     }
+
+    fn stream_records_with_callback(&self, callback: impl FnMut(StateRecord)) -> io::Result<()> {
+        let reader = BufReader::new(File::open(&self.records_file)?);
+        stream_records_from_file(reader, callback).map_err(io::Error::from)
+    }
+
+    /// If records vector is empty processes records stream from records_file.
+    /// May panic if records_file is removed or is in wrong format.
+    pub fn for_each_record(&self, mut callback: impl FnMut(&StateRecord)) {
+        if self.records.as_ref().is_empty() {
+            let callback_move = |record: StateRecord| {
+                callback(&record);
+            };
+            self.stream_records_with_callback(callback_move)
+                .expect("error while streaming records");
+        } else {
+            for record in self.records.as_ref() {
+                callback(record);
+            }
+        }
+    }
 }
 
 pub fn get_initial_supply(records: &[StateRecord]) -> Balance {

core/primitives/src/state_record.rs

@@ -147,3 +147,16 @@ fn to_printable(blob: &[u8]) -> String {
         }
     }
 }
+
+pub fn state_record_to_account_id(state_record: &StateRecord) -> &AccountId {
+    match state_record {
+        StateRecord::Account { account_id, .. }
+        | StateRecord::AccessKey { account_id, .. }
+        | StateRecord::Contract { account_id, .. }
+        | StateRecord::ReceivedData { account_id, .. }
+        | StateRecord::Data { account_id, .. } => account_id,
+        StateRecord::PostponedReceipt(receipt) | StateRecord::DelayedReceipt(receipt) => {
+            &receipt.receiver_id
+        }
+    }
+}

neard/Cargo.toml

@@ -7,9 +7,9 @@ default-run = "neard"
 
 [dependencies]
 jemallocator = { version = "0.3"  }
+awc = "3.0.0-beta.5"
 actix = "0.11.0-beta.2"
 actix-rt = "2"
-actix-web = { version = "4.0.0-beta.1", features = [ "openssl" ] }
 byteorder = "1.2"
 easy-ext = "0.2"
 chrono = { version = "0.4.4", features = ["serde"] }