chain: ShardManager: add metrics for partial encoded chunk response (n…

…ear#6431)

This is commit e92e894 upstream.

Add near_partial_encoded_chunk_request_processing_time metric which
returns how much time processing partial encoded chunk requests took.
The metric is split by the method used to create a response and also
whether in the end the response has been prepared or not.

Issue: near#6242

chain/chunks/Cargo.toml

@@ -16,12 +16,15 @@ log = "0.4"
 borsh = "0.9"
 lru = "0.7.2"
 reed-solomon-erasure = "4"
+once_cell = "1.5.2"
+prometheus = "0.11"
 
 near-crypto = { path = "../../core/crypto" }
 near-primitives = { path = "../../core/primitives" }
 near-chunks-primitives = { path = "../chunks-primitives" }
 near-store = { path = "../../core/store" }
 near-network = { path = "../network" }
+near-metrics = { path = "../../core/metrics" }
 near-chain = { path = "../chain" }
 near-pool = { path = "../pool" }
 near-network-primitives = { path = "../network-primitives" }

chain/chunks/src/lib.rs

@@ -127,6 +127,7 @@ use near_primitives::epoch_manager::RngSeed;
 use rand::Rng;
 
 mod chunk_cache;
+mod metrics;
 pub mod test_utils;
 
 const CHUNK_PRODUCER_BLACKLIST_SIZE: usize = 100;
@@ -999,32 +1000,62 @@ impl ShardsManager {
     ) {
         debug!(target: "chunks", "Received partial encoded chunk request for {:?}, part_ordinals: {:?}, shards: {:?}, I'm {:?}", request.chunk_hash.0, request.part_ords, request.tracking_shards, self.me);
 
-        let response = if let Some(entry) = self.encoded_chunks.get(&request.chunk_hash) {
-            Self::prepare_partial_encoded_chunk_response_from_cache(request, entry)
-        } else if let Ok(partial_chunk) = chain_store.get_partial_chunk(&request.chunk_hash) {
-            Self::prepare_partial_encoded_chunk_response_from_partial(request, partial_chunk)
-        } else if let Ok(chunk) = chain_store.get_chunk(&request.chunk_hash).map(|ch| ch.clone()) {
+        let (started, key, response) =
+            self.prepare_partial_encoded_chunk_response(request, chain_store, rs);
+
+        let elapsed = started.elapsed().as_secs_f64();
+        let labels = [key, if response.is_some() { "ok" } else { "failed" }];
+        metrics::PARTIAL_ENCODED_CHUNK_REQUEST_PROCESSING_TIME
+            .with_label_values(&labels)
+            .observe(elapsed);
+
+        if let Some(response) = response {
+            self.peer_manager_adapter.do_send(PeerManagerMessageRequest::NetworkRequests(
+                NetworkRequests::PartialEncodedChunkResponse { route_back, response },
+            ))
+        }
+    }
+
+    fn prepare_partial_encoded_chunk_response(
+        &mut self,
+        request: PartialEncodedChunkRequestMsg,
+        chain_store: &mut ChainStore,
+        rs: &mut ReedSolomonWrapper,
+    ) -> (std::time::Instant, &'static str, Option<PartialEncodedChunkResponseMsg>) {
+        // Try getting data from in-memory cache.
+        let started = Instant::now();
+        if let Some(entry) = self.encoded_chunks.get(&request.chunk_hash) {
+            let response = Self::prepare_partial_encoded_chunk_response_from_cache(request, entry);
+            return (started, "cache", response);
+        }
+
+        // Try fetching partial encoded chunk from storage.
+        let started = Instant::now();
+        if let Ok(partial_chunk) = chain_store.get_partial_chunk(&request.chunk_hash) {
+            let response =
+                Self::prepare_partial_encoded_chunk_response_from_partial(request, partial_chunk);
+            return (started, "partial", response);
+        }
+
+        // Try fetching chunk from storage and recomputing encoded chunk from
+        // it.  If we are archival node we might have garbage collected the
+        // partial chunk while we still keep the chunk itself.  We can get the
+        // chunk, recalculate the parts and respond to the request.
+        let started = Instant::now();
+        if let Ok(chunk) = chain_store.get_chunk(&request.chunk_hash).map(|ch| ch.clone()) {
             // Note: we need to clone the chunk because otherwise we would be
             // holding multiple references to chain_store.  One through the
             // chunk and another through chain_store which we need to pass down
             // to do further fetches.
 
-            // If we are archival node we might have garbage collected the
-            // partial chunk while we still keep the chunk itself.  We can get
-            // the chunk, recalculate the parts and respond to the request.
-            //
             // TODO(#6242): This is currently not implemented and effectively
             // this is dead code.
-            self.prepare_partial_encoded_chunk_response_from_chunk(request, rs, chunk)
-        } else {
-            None
-        };
-
-        if let Some(response) = response {
-            self.peer_manager_adapter.do_send(PeerManagerMessageRequest::NetworkRequests(
-                NetworkRequests::PartialEncodedChunkResponse { route_back, response },
-            ))
+            let response =
+                self.prepare_partial_encoded_chunk_response_from_chunk(request, rs, chunk);
+            return (started, "chunk", response);
         }
+
+        (Instant::now(), "none", None)
     }
 
     /// Prepares response to a partial encoded chunk request from an entry in

chain/chunks/src/metrics.rs

@@ -0,0 +1,20 @@
+use once_cell::sync::Lazy;
+
+pub static PARTIAL_ENCODED_CHUNK_REQUEST_PROCESSING_TIME: Lazy<near_metrics::HistogramVec> =
+    Lazy::new(|| {
+        near_metrics::try_create_histogram_vec(
+            "near_partial_encoded_chunk_request_processing_time",
+            concat!(
+                "Time taken to prepare responses to partial encoded chuck ",
+                "requests.  The ‘method’ key describes how we tried to fulfil ",
+                "the request and ‘success’ describes whether we managed to ",
+                "create a response (‘ok’) or not (‘failed’).  Note that ",
+                "success does not mean that we managed to send the response ",
+                "over network; the count is taken before we attempt to send ",
+                "the data out."
+            ),
+            &["method", "success"],
+            Some(prometheus::exponential_buckets(0.001, 2.0, 16).unwrap()),
+        )
+        .unwrap()
+    });