[CherryPick 1.18] Add settlement finality latency metric (#16155)

## Description 

This PR adds a new metric to track transaction settlement finality. This
is more accurate because it only tracks the case where a transaction is
driven from raw transaction to effects with one try, successfully.
Also use custom buckets and built-in histogram as it's more accurate.

## Test Plan 

CI

---
If your changes are not user-facing and do not break anything, you can
skip the following section. Otherwise, please briefly describe what has
changed under the Release Notes section.

### Type of Change (Check all that apply)

- [ ] protocol change
- [ ] user-visible impact
- [ ] breaking change for a client SDKs
- [ ] breaking change for FNs (FN binary must upgrade)
- [ ] breaking change for validators or node operators (must upgrade
binaries)
- [ ] breaking change for on-chain data layout
- [ ] necessitate either a data wipe or data migration

### Release notes

crates/sui-core/src/quorum_driver/metrics.rs

@@ -3,12 +3,21 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use prometheus::{
-    register_int_counter_vec_with_registry, register_int_counter_with_registry,
-    register_int_gauge_with_registry, IntCounter, IntCounterVec, IntGauge, Registry,
+    register_histogram_vec_with_registry, register_int_counter_vec_with_registry,
+    register_int_counter_with_registry, register_int_gauge_with_registry, HistogramVec, IntCounter,
+    IntCounterVec, IntGauge, Registry,
 };
 
 use mysten_metrics::histogram::Histogram;
 
+const FINALITY_LATENCY_SEC_BUCKETS: &[f64] = &[
+    0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85,
+    0.9, 0.95, 1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2.0, 2.1, 2.2, 2.3, 2.4, 2.5, 2.6,
+    2.7, 2.8, 2.9, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9, 4.0, 4.5, 5.0, 5.5, 6.0, 6.5,
+    7.0, 7.5, 8.0, 8.5, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0,
+    25.0,
+];
+
 #[derive(Clone)]
 pub struct QuorumDriverMetrics {
     pub(crate) total_requests: IntCounter,
@@ -24,6 +33,8 @@ pub struct QuorumDriverMetrics {
     pub(crate) total_attempts_retrying_conflicting_transaction: IntCounter,
     pub(crate) total_successful_attempts_retrying_conflicting_transaction: IntCounter,
     pub(crate) total_times_conflicting_transaction_already_finalized_when_retrying: IntCounter,
+
+    pub(crate) settlement_finality_latency: HistogramVec,
 }
 
 impl QuorumDriverMetrics {
@@ -89,6 +100,14 @@ impl QuorumDriverMetrics {
                 registry,
             )
             .unwrap(),
+            settlement_finality_latency: register_histogram_vec_with_registry!(
+                "quorum_driver_settlement_finality_latency",
+                "Settlement finality latency observed from quorum driver",
+                &["tx_type"],
+                FINALITY_LATENCY_SEC_BUCKETS.to_vec(),
+                registry,
+            )
+            .unwrap(),
         }
     }
 

crates/sui-core/src/quorum_driver/mod.rs

@@ -31,7 +31,9 @@ use crate::authority_aggregator::{
 };
 use crate::authority_client::AuthorityAPI;
 use mysten_common::sync::notify_read::{NotifyRead, Registration};
-use mysten_metrics::{spawn_monitored_task, GaugeGuard};
+use mysten_metrics::{
+    spawn_monitored_task, GaugeGuard, TX_TYPE_SHARED_OBJ_TX, TX_TYPE_SINGLE_WRITER_TX,
+};
 use std::fmt::Write;
 use sui_types::error::{SuiError, SuiResult};
 use sui_types::messages_safe_client::PlainTransactionInfoResponse;
@@ -683,7 +685,9 @@ where
             ..
         } = task;
         let tx_digest = *transaction.digest();
+        let is_single_writer_tx = !transaction.contains_shared_object();
 
+        let timer = Instant::now();
         let tx_cert = match tx_cert {
             None => match quorum_driver.process_transaction(transaction.clone()).await {
                 Ok(ProcessTransactionResult::Certified(tx_cert)) => {
@@ -736,6 +740,15 @@ where
                 return;
             }
         };
+        quorum_driver
+            .metrics
+            .settlement_finality_latency
+            .with_label_values(&[if is_single_writer_tx {
+                TX_TYPE_SINGLE_WRITER_TX
+            } else {
+                TX_TYPE_SHARED_OBJ_TX
+            }])
+            .observe(timer.elapsed().as_secs_f64());
 
         quorum_driver.notify(&transaction, &Ok(response), old_retry_times + 1);
     }