diff --git a/dashboards/lodestar_state_cache_regen.json b/dashboards/lodestar_state_cache_regen.json index 62b4f6fc479f..451130639f47 100644 --- a/dashboards/lodestar_state_cache_regen.json +++ b/dashboards/lodestar_state_cache_regen.json @@ -91,6 +91,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -200,6 +201,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -309,6 +311,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -389,6 +392,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -481,6 +485,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -561,6 +566,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -642,6 +648,9 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -653,6 +662,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -790,6 +800,9 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -801,6 +814,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -938,6 +952,9 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -949,6 +966,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1087,6 +1105,9 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -1098,6 +1119,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1225,6 +1247,457 @@ "title": "CP state cache - seconds since last read per state entry", "type": "timeseries" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 58, + "panels": [], + "title": "n-historical states", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 42 + }, + "id": 56, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_cp_state_cache_state_serialize_seconds_sum[$rate_interval])\n/\nrate(lodestar_cp_state_cache_state_serialize_seconds_count[$rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "serialize_duration", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_cp_state_cache_state_persist_seconds_from_slot_sum[$rate_interval])\n/\nrate(lodestar_cp_state_cache_state_persist_seconds_from_slot_count[$rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "sec_from_slot", + "range": true, + "refId": "A" + } + ], + "title": "State persist", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "epoch_diff" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 42 + }, + "id": 55, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_cp_state_cache_state_reload_seconds_sum[$rate_interval])\n/\nrate(lodestar_cp_state_cache_state_reload_seconds_count[$rate_interval])", + "instant": false, + "legendFormat": "reload_duration", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_cp_state_cache_state_reload_seconds_from_slot_sum[$rate_interval])\n/\nrate(lodestar_cp_state_cache_state_reload_seconds_from_slot_count[$rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "sec_from_slot", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_cp_state_cache_state_reload_db_read_seconds_sum[$rate_interval])\n/\nrate(lodestar_cp_state_cache_state_reload_db_read_seconds_count[$rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "db_read_duration", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_cp_state_cache_state_reload_epoch_diff_sum[$rate_interval])\n/\nrate(lodestar_cp_state_cache_state_reload_epoch_diff_count[$rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "epoch_diff", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_cp_state_cache_state_reload_validators_serialize_seconds_sum[$rate_interval])\n/\nrate(lodestar_cp_state_cache_state_reload_validators_serialize_seconds_count[$rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "validators_serialization_duration", + "range": true, + "refId": "E" + } + ], + "title": "Reload duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 50 + }, + "id": 59, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_cp_state_cache_state_prune_from_memory_count[$rate_interval]) * 384", + "instant": false, + "legendFormat": "from_memory", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_cp_state_cache_persisted_state_remove_count[$rate_interval]) * 384", + "hide": false, + "instant": false, + "legendFormat": "from_persistent", + "range": true, + "refId": "B" + } + ], + "title": "State prune per epoch", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 50 + }, + "id": 57, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_cp_state_cache_persisted_state_alloc_count[$rate_interval])", + "instant": false, + "legendFormat": "state_persist", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_cp_state_cache_state_reload_validators_serialize_alloc_count[$rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "state_reload_validator_serialization", + "range": true, + "refId": "B" + } + ], + "title": "Alloc count", + "type": "timeseries" + }, { "collapsed": false, "datasource": { @@ -1235,7 +1708,7 @@ "h": 1, "w": 24, "x": 0, - "y": 41 + "y": 58 }, "id": 40, "panels": [], @@ -1263,6 +1736,9 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -1274,6 +1750,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1298,7 +1775,7 @@ "h": 8, "w": 12, "x": 0, - "y": 42 + "y": 59 }, "id": 24, "options": { @@ -1340,6 +1817,9 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -1351,6 +1831,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1375,7 +1856,7 @@ "h": 8, "w": 12, "x": 12, - "y": 42 + "y": 59 }, "id": 26, "options": { @@ -1417,6 +1898,9 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -1428,6 +1912,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1453,7 +1938,7 @@ "h": 8, "w": 12, "x": 0, - "y": 50 + "y": 67 }, "id": 28, "options": { @@ -1495,6 +1980,9 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -1506,6 +1994,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1531,7 +2020,7 @@ "h": 8, "w": 12, "x": 12, - "y": 50 + "y": 67 }, "id": 30, "options": { @@ -1573,6 +2062,9 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -1584,6 +2076,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1611,7 +2104,7 @@ "h": 7, "w": 6, "x": 0, - "y": 58 + "y": 75 }, "id": 32, "options": { @@ -1653,6 +2146,9 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -1664,6 +2160,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1691,7 +2188,7 @@ "h": 7, "w": 6, "x": 6, - "y": 58 + "y": 75 }, "id": 36, "options": { @@ -1733,6 +2230,9 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -1744,6 +2244,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1771,7 +2272,7 @@ "h": 7, "w": 6, "x": 12, - "y": 58 + "y": 75 }, "id": 34, "options": { @@ -1813,6 +2314,9 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -1824,6 +2328,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1851,7 +2356,7 @@ "h": 7, "w": 6, "x": 18, - "y": 58 + "y": 75 }, "id": 38, "options": { @@ -1892,7 +2397,7 @@ "h": 1, "w": 24, "x": 0, - "y": 65 + "y": 82 }, "id": 54, "panels": [], @@ -1919,6 +2424,9 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -1931,6 +2439,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1956,7 +2465,7 @@ "h": 7, "w": 12, "x": 0, - "y": 66 + "y": 83 }, "id": 42, "options": { @@ -2000,6 +2509,9 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -2012,6 +2524,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2037,7 +2550,7 @@ "h": 7, "w": 6, "x": 12, - "y": 66 + "y": 83 }, "id": 44, "options": { @@ -2080,6 +2593,9 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -2092,6 +2608,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2117,7 +2634,7 @@ "h": 7, "w": 6, "x": 18, - "y": 66 + "y": 83 }, "id": 48, "options": { @@ -2160,6 +2677,9 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -2172,6 +2692,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2197,7 +2718,7 @@ "h": 7, "w": 12, "x": 0, - "y": 73 + "y": 90 }, "id": 46, "options": { @@ -2240,6 +2761,9 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -2252,6 +2776,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2277,7 +2802,7 @@ "h": 7, "w": 6, "x": 12, - "y": 73 + "y": 90 }, "id": 50, "options": { @@ -2320,6 +2845,9 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -2332,6 +2860,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2357,7 +2886,7 @@ "h": 7, "w": 6, "x": 18, - "y": 73 + "y": 90 }, "id": 52, "options": { @@ -2391,8 +2920,7 @@ } ], "refresh": "10s", - "schemaVersion": 38, - "style": "dark", + "schemaVersion": 39, "tags": [ "lodestar" ], diff --git a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts index 3b38123de838..e33407a8dca6 100644 --- a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts +++ b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts @@ -203,13 +203,13 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { try { // 80% of validators serialization time comes from memory allocation, this is to avoid it - const sszTimer = this.metrics?.stateReloadValidatorsSszDuration.startTimer(); + const sszTimer = this.metrics?.stateReloadValidatorsSerializeDuration.startTimer(); // automatically free the buffer pool after this scope using validatorsBytesWithKey = this.serializeStateValidators(seedState); let validatorsBytes = validatorsBytesWithKey?.buffer; if (validatorsBytes == null) { // fallback logic in case we can't use the buffer pool - this.metrics?.stateReloadValidatorsSszAllocCount.inc(); + this.metrics?.stateReloadValidatorsSerializeAllocCount.inc(); validatorsBytes = seedState.validators.serialize(); } sszTimer?.(); diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index e6b4d2f5a683..20dc0333989a 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -1209,13 +1209,13 @@ export function createLodestarMetrics( help: "Histogram of time to persist state to db since the clock slot", buckets: [0, 2, 4, 6, 8, 10, 12], }), - stateReloadValidatorsSszDuration: register.histogram({ - name: "lodestar_cp_state_cache_state_reload_validators_ssz_seconds", + stateReloadValidatorsSerializeDuration: register.histogram({ + name: "lodestar_cp_state_cache_state_reload_validators_serialize_seconds", help: "Histogram of time to serialize validators", buckets: [0.1, 0.2, 0.5, 1], }), - stateReloadValidatorsSszAllocCount: register.counter({ - name: "lodestar_cp_state_cache_state_reload_validators_ssz_alloc_count", + stateReloadValidatorsSerializeAllocCount: register.counter({ + name: "lodestar_cp_state_cache_state_reload_validators_serialize_alloc_count", help: "Total number time to allocate memory for validators serialization", }), stateReloadShufflingCacheMiss: register.counter({