Skip to content

Commit

Permalink
fix(inx): dedicated analytics_time in sync_time metrics (#888)
Browse files Browse the repository at this point in the history 
* Use transactions where possible and make analytics "sync"

* Revert transactions

* Add analytics timings

* fix dashboard

Co-authored-by: Jochen Görtler <jochen.goertler@iota.org>
  • Loading branch information
@grtlr
Alexandcoats and grtlr authored Nov 16, 2022
1 parent 86d87b3 commit e94c171
Show file tree
Hide file tree
Showing 3 changed files with 228 additions and 35 deletions.
227 changes: 210 additions & 17 deletions docker/assets/grafana/dashboards/chronicle_dashboard.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@
},
"textMode": "auto"
},
"pluginVersion": "9.1.6",
"pluginVersion": "9.2.4",
"targets": [
{
"datasource": {
Expand Down Expand Up @@ -201,25 +201,26 @@
"pluginVersion": "9.2.3",
"targets": [
{
"alias": "Network: \"$tag_network_name\", Chronicle Version: $tag_chronicle_version",
"alias": "Milestone Time",
"datasource": {
"type": "influxdb",
"uid": "P951FEA4DE68E13C5"
},
"groupBy": [
{
"params": [
"chronicle_version"
"$__interval"
],
"type": "tag"
"type": "time"
},
{
"params": [
"network_name"
"null"
],
"type": "tag"
"type": "fill"
}
],
"hide": false,
"measurement": "sync_metrics",
"orderByTime": "ASC",
"policy": "default",
Expand All @@ -229,9 +230,55 @@
[
{
"params": [
"sync_time"
"milestone_time"
],
"type": "field"
},
{
"params": [],
"type": "mean"
}
]
],
"tags": []
},
{
"alias": "Analytics Time",
"datasource": {
"type": "influxdb",
"uid": "P951FEA4DE68E13C5"
},
"groupBy": [
{
"params": [
"$__interval"
],
"type": "time"
},
{
"params": [
"null"
],
"type": "fill"
}
],
"hide": false,
"measurement": "sync_metrics",
"orderByTime": "ASC",
"policy": "default",
"refId": "B",
"resultFormat": "time_series",
"select": [
[
{
"params": [
"analytics_time"
],
"type": "field"
},
{
"params": [],
"type": "mean"
}
]
],
Expand All @@ -241,6 +288,128 @@
"title": "Milestone Sync Time",
"type": "timeseries"
},
{
"datasource": {
"type": "influxdb",
"uid": "P951FEA4DE68E13C5"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 11,
"w": 24,
"x": 0,
"y": 15
},
"id": 15,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"alias": "Last Milestone Index",
"datasource": {
"type": "influxdb",
"uid": "P951FEA4DE68E13C5"
},
"groupBy": [
{
"params": [
"$__interval"
],
"type": "time"
},
{
"params": [
"null"
],
"type": "fill"
}
],
"measurement": "sync_metrics",
"orderByTime": "ASC",
"policy": "default",
"refId": "A",
"resultFormat": "time_series",
"select": [
[
{
"params": [
"milestone_index"
],
"type": "field"
},
{
"params": [],
"type": "last"
}
]
],
"tags": []
}
],
"title": "Milestone Index over Time",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
Expand Down Expand Up @@ -303,7 +472,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 15
"y": 26
},
"id": 9,
"options": {
Expand Down Expand Up @@ -396,7 +565,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 15
"y": 26
},
"id": 11,
"options": {
Expand All @@ -412,14 +581,26 @@
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "mongodb_dbstats_indexSize{database=\"chronicle\"}",
"hide": false,
"legendFormat": "Total database: {{ database }}",
"range": true,
"refId": "D"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "mongodb_collstats_storageStats_indexSizes_ledger_update_index",
"legendFormat": "{{ collection }}",
"legendFormat": "Ledger Update Index",
"range": true,
"refId": "A"
},
Expand All @@ -443,7 +624,7 @@
"editorMode": "code",
"expr": "mongodb_collstats_storageStats_indexSizes_block_referenced_index",
"hide": false,
"legendFormat": "{{ collection }}",
"legendFormat": "Referenced Block Index",
"range": true,
"refId": "C"
},
Expand All @@ -453,11 +634,23 @@
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "mongodb_dbstats_indexSize{database=\"chronicle\"}",
"expr": "mongodb_collstats_storageStats_indexSizes_output_booked_milestone_index",
"hide": false,
"legendFormat": "{{ database }}",
"legendFormat": "Output Booked Index",
"range": true,
"refId": "D"
"refId": "E"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "mongodb_collstats_storageStats_indexSizes_id",
"hide": false,
"legendFormat": "Id Index: {{ collection }}",
"range": true,
"refId": "F"
}
],
"title": "MongoDB Index Info",
Expand Down Expand Up @@ -525,7 +718,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 23
"y": 34
},
"id": 13,
"options": {
Expand Down Expand Up @@ -565,7 +758,7 @@
"list": []
},
"time": {
"from": "now-30m",
"from": "now-3h",
"to": "now"
},
"timepicker": {},
Expand All @@ -574,4 +767,4 @@
"uid": "up-bdCiVz",
"version": 1,
"weekStart": ""
}
}
32 changes: 15 additions & 17 deletions src/bin/inx-chronicle/stardust_inx/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -329,6 +329,18 @@ impl InxWorker {
#[allow(unused)]
let milestone_timestamp = self.handle_milestone(inx, milestone_index).await?;

#[cfg(all(feature = "analytics", feature = "metrics"))]
let analytics_start_time = std::time::Instant::now();
#[cfg(feature = "analytics")]
if let Some(influx_db) = &self.influx_db {
let analytics = self.db.get_all_analytics(milestone_index).await?;
influx_db
.insert_all_analytics(milestone_timestamp, milestone_index, analytics)
.await?;
}
#[cfg(all(feature = "analytics", feature = "metrics"))]
let analytics_elapsed = analytics_start_time.elapsed();

#[cfg(feature = "metrics")]
if let Some(influx_db) = &self.influx_db {
// Unwrap: Safe because we checked above
Expand All @@ -337,7 +349,9 @@ impl InxWorker {
.insert(chronicle::db::collections::metrics::SyncMetrics {
time: chrono::Utc::now(),
milestone_index,
sync_time: elapsed.as_millis() as u64,
milestone_time: elapsed.as_millis() as u64,
#[cfg(feature = "analytics")]
analytics_time: analytics_elapsed.as_millis() as u64,
network_name,
chronicle_version: std::env!("CARGO_PKG_VERSION").to_string(),
})
Expand Down Expand Up @@ -397,22 +411,6 @@ impl InxWorker {

let milestone_timestamp = milestone.milestone_info.milestone_timestamp.into();

#[cfg(feature = "analytics")]
if let Some(influx_db) = &self.influx_db {
let db = self.db.clone();
let influx_db = influx_db.clone();
tokio::spawn(async move {
let analytics = db.get_all_analytics(milestone_index).await?;
influx_db
.insert_all_analytics(milestone_timestamp, milestone_index, analytics)
.await?;

tracing::debug!("Finished analytics for milestone: {}", milestone_index);

Result::<_, InxWorkerError>::Ok(())
});
}

let milestone_id = milestone
.milestone_info
.milestone_id
Expand Down
4 changes: 3 additions & 1 deletion src/db/collections/metrics/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,9 @@ use crate::types::tangle::MilestoneIndex;
pub struct SyncMetrics {
pub time: DateTime<Utc>,
pub milestone_index: MilestoneIndex,
pub sync_time: u64,
pub milestone_time: u64,
#[cfg(feature = "analytics")]
pub analytics_time: u64,
#[cfg_attr(feature = "influxdb", influxdb(tag))]
pub chronicle_version: String,
#[cfg_attr(feature = "influxdb", influxdb(tag))]
Expand Down

0 comments on commit e94c171

Please sign in to comment.