Skip to content

Commit

Permalink
Merge pull request #1 from holepunchto/more-hypercore-stats
Browse files Browse the repository at this point in the history
More hypercore stats
  • Loading branch information
HDegroote authored Sep 2, 2024
2 parents 48d16e1 + 9b61715 commit 7f37735
Show file tree
Hide file tree
Showing 3 changed files with 304 additions and 2 deletions.
268 changes: 268 additions & 0 deletions index.js
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,86 @@ class HypercoreStats {
return this._getStats().totalMaxInflightBlocks
}

// getTotalBlocksUploaded () {
// return this._getStats().totalBlocksUploaded
// }

// getTotalBlocksDownloaded () {
// return this._getStats().totalBlocksDownloaded
// }

// getTotalBytesUploaded () {
// return this._getStats().totalBytesUploaded
// }

// getTotalBytesDownloaded () {
// return this._getStats().totalBytesDownloaded
// }

get totalWireSyncReceived () {
return this._getStats().totalWireSyncReceived
}

get totalWireSyncTransmitted () {
return this._getStats().totalWireSyncTransmitted
}

get totalWireRequestReceived () {
return this._getStats().totalWireRequestReceived
}

get totalWireRequestTransmitted () {
return this._getStats().totalWireRequestTransmitted
}

get totalWireCancelReceived () {
return this._getStats().totalWireCancelReceived
}

get totalWireCancelTransmitted () {
return this._getStats().totalWireCancelTransmitted
}

get totalWireDataReceived () {
return this._getStats().totalWireDataReceived
}

get totalWireDataTransmitted () {
return this._getStats().totalWireDataTransmitted
}

get totalWireWantReceived () {
return this._getStats().totalWireWantReceived
}

get totalWireWantTransmitted () {
return this._getStats().totalWireWantTransmitted
}

get totalWireBitfieldReceived () {
return this._getStats().totalWireBitfieldReceived
}

get totalWireBitfieldTransmitted () {
return this._getStats().totalWireBitfieldTransmitted
}

get totalWireRangeReceived () {
return this._getStats().totalWireRangeReceived
}

get totalWireRangeTransmitted () {
return this._getStats().totalWireRangeTransmitted
}

get totalWireExtensionReceived () {
return this._getStats().totalWireExtensionReceived
}

get totalWireExtensionTransmitted () {
return this._getStats().totalWireExtensionTransmitted
}

// Caches the result for this._lastStatsCalcTime ms
_getStats () {
if (this._cachedStats && this._lastStatsCalcTime + this.cacheExpiryMs > Date.now()) {
Expand Down Expand Up @@ -99,6 +179,149 @@ class HypercoreStats {
this.set(self.getTotalPeers())
}
})

/*
new promClient.Gauge({ // eslint-disable-line no-new
name: 'hypercore_total_blocks_uploaded',
help: 'Total amount of blocks uploaded across all cores',
collect () {
this.set(self.getTotalBlocksUploaded())
}
})
new promClient.Gauge({ // eslint-disable-line no-new
name: 'hypercore_total_blocks_downloaded',
help: 'Total amount of blocks downloaded across all cores',
collect () {
this.set(self.getTotalBlocksDownloaded())
}
})
new promClient.Gauge({ // eslint-disable-line no-new
name: 'hypercore_total_bytes_uploaded',
help: 'Total amount of bytes uploaded across all cores',
collect () {
this.set(self.getTotalBytesUploaded())
}
})
new promClient.Gauge({ // eslint-disable-line no-new
name: 'hypercore_total_bytes_downloaded',
help: 'Total amount of bytes downloaded across all cores',
collect () {
this.set(self.getTotalBytesDownloaded())
}
}) */

new promClient.Gauge({ // eslint-disable-line no-new
name: 'hypercore_total_wire_sync_received',
help: 'Total amount of wire-sync messages received across all cores',
collect () {
this.set(self.totalWireSyncReceived)
}
})
new promClient.Gauge({ // eslint-disable-line no-new
name: 'hypercore_total_wire_sync_transmitted',
help: 'Total amount of wire-sync messages transmitted across all cores',
collect () {
this.set(self.totalWireSyncTransmitted)
}
})
new promClient.Gauge({ // eslint-disable-line no-new
name: 'hypercore_total_wire_request_received',
help: 'Total amount of wire-request messages received across all cores',
collect () {
this.set(self.totalWireRequestReceived)
}
})
new promClient.Gauge({ // eslint-disable-line no-new
name: 'hypercore_total_wire_request_transmitted',
help: 'Total amount of wire-request messages transmitted across all cores',
collect () {
this.set(self.totalWireRequestTransmitted)
}
})
new promClient.Gauge({ // eslint-disable-line no-new
name: 'hypercore_total_wire_cancel_received',
help: 'Total amount of wire-cancel messages received across all cores',
collect () {
this.set(self.totalWireCancelReceived)
}
})
new promClient.Gauge({ // eslint-disable-line no-new
name: 'hypercore_total_wire_cancel_transmitted',
help: 'Total amount of wire-cancel messages transmitted across all cores',
collect () {
this.set(self.totalWireCancelTransmitted)
}
})
new promClient.Gauge({ // eslint-disable-line no-new
name: 'hypercore_total_wire_data_received',
help: 'Total amount of wire-data messages received across all cores',
collect () {
this.set(self.totalWireDataReceived)
}
})
new promClient.Gauge({ // eslint-disable-line no-new
name: 'hypercore_total_wire_data_transmitted',
help: 'Total amount of wire-data messages transmitted across all cores',
collect () {
this.set(self.totalWireDataTransmitted)
}
})
new promClient.Gauge({ // eslint-disable-line no-new
name: 'hypercore_total_wire_want_received',
help: 'Total amount of wire-want messages received across all cores',
collect () {
this.set(self.totalWireWantReceived)
}
})
new promClient.Gauge({ // eslint-disable-line no-new
name: 'hypercore_total_wire_want_transmitted',
help: 'Total amount of wire-want messages transmitted across all cores',
collect () {
this.set(self.totalWireWantTransmitted)
}
})
new promClient.Gauge({ // eslint-disable-line no-new
name: 'hypercore_total_wire_bitfield_received',
help: 'Total amount of wire-bitfield messages received across all cores',
collect () {
this.set(self.totalWireBitfieldReceived)
}
})
new promClient.Gauge({ // eslint-disable-line no-new
name: 'hypercore_total_wire_bitfield_transmitted',
help: 'Total amount of wire-bitfield messages transmitted across all cores',
collect () {
this.set(self.totalWireBitfieldTransmitted)
}
})
new promClient.Gauge({ // eslint-disable-line no-new
name: 'hypercore_total_wire_range_received',
help: 'Total amount of wire-range messages received across all cores',
collect () {
this.set(self.totalWireRangeReceived)
}
})
new promClient.Gauge({ // eslint-disable-line no-new
name: 'hypercore_total_wire_range_transmitted',
help: 'Total amount of wire-range messages transmitted across all cores',
collect () {
this.set(self.totalWireRangeTransmitted)
}
})
new promClient.Gauge({ // eslint-disable-line no-new
name: 'hypercore_total_wire_extension_received',
help: 'Total amount of wire-extension messages received across all cores',
collect () {
this.set(self.totalWireExtensionReceived)
}
})
new promClient.Gauge({ // eslint-disable-line no-new
name: 'hypercore_total_wire_extension_transmitted',
help: 'Total amount of wire-extension messages transmitted across all cores',
collect () {
this.set(self.totalWireExtensionTransmitted)
}
})
}
}

Expand All @@ -107,10 +330,32 @@ class HypercoreStatsSnapshot {
this.cores = cores

this._totalPeersConns = new Set()

this.totalWireSyncReceived = 0
this.totalWireSyncTransmitted = 0
this.totalWireRequestReceived = 0
this.totalWireRequestTransmitted = 0
this.totalWireCancelReceived = 0
this.totalWireCancelTransmitted = 0
this.totalWireDataReceived = 0
this.totalWireDataTransmitted = 0
this.totalWireWantReceived = 0
this.totalWireWantTransmitted = 0
this.totalWireBitfieldReceived = 0
this.totalWireBitfieldTransmitted = 0
this.totalWireRangeReceived = 0
this.totalWireRangeTransmitted = 0
this.totalWireExtensionReceived = 0
this.totalWireExtensionTransmitted = 0

this.totalCores = 0
this.totalLength = 0
this.totalInflightBlocks = 0
this.totalMaxInflightBlocks = 0
// this.totalBlocksUploaded = 0
// this.totalBlocksDownloaded = 0
// this.totalBytesUploaded = 0
// this.totalBytesDownloaded = 0

this.calculate()
}
Expand All @@ -124,6 +369,29 @@ class HypercoreStatsSnapshot {

for (const core of this.cores) {
this.totalLength += core.length
// this.totalBlocksUploaded += core.stats.blocksUploaded
// this.totalBlocksDownloaded += core.stats.blocksDownloaded
// this.totalBytesUploaded += core.stats.bytesUploaded
// this.totalBytesDownloaded += core.stats.bytesDownloaded

if (core.replicator) {
this.totalWireSyncReceived += core.replicator.stats.wireSync.rx
this.totalWireSyncTransmitted += core.replicator.stats.wireSync.tx
this.totalWireRequestReceived += core.replicator.stats.wireRequest.rx
this.totalWireRequestTransmitted += core.replicator.stats.wireRequest.tx
this.totalWireCancelReceived += core.replicator.stats.wireCancel.rx
this.totalWireCancelTransmitted += core.replicator.stats.wireCancel.tx
this.totalWireDataReceived += core.replicator.stats.wireData.rx
this.totalWireDataTransmitted += core.replicator.stats.wireData.tx
this.totalWireWantReceived += core.replicator.stats.wireWant.rx
this.totalWireWantTransmitted += core.replicator.stats.wireWant.tx
this.totalWireBitfieldReceived += core.replicator.stats.wireBitfield.rx
this.totalWireBitfieldTransmitted += core.replicator.stats.wireBitfield.tx
this.totalWireRangeReceived += core.replicator.stats.wireRange.rx
this.totalWireRangeTransmitted += core.replicator.stats.wireRange.tx
this.totalWireExtensionReceived += core.replicator.stats.wireExtension.rx
this.totalWireExtensionTransmitted += core.replicator.stats.wireExtension.tx
}

for (const peer of core.peers) {
this.totalInflightBlocks += peer.inflight
Expand Down
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
"devDependencies": {
"brittle": "^3.6.1",
"corestore": "^6.18.4",
"hypercore": "^10.37.18",
"hypercore": "^10.37.19",
"prom-client": "^15.1.3",
"standard": "^17.1.0"
},
Expand Down
36 changes: 35 additions & 1 deletion test.js
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,28 @@ test('Can register and get prometheus metrics', async (t) => {
t.is(getMetricValue(lines, 'hypercore_total_inflight_blocks'), 0, 'hypercore_total_inflight_blocks init 0')
t.is(getMetricValue(lines, 'hypercore_total_max_inflight_blocks'), 0, 'hypercore_total_max_inflight_blocks init 0')
t.is(getMetricValue(lines, 'hypercore_total_peers'), 0, 'hypercore_total_peers init 0')

// t.is(getMetricValue(lines, 'hypercore_total_blocks_downloaded'), 0, 'hypercore_total_blocks_downloaded init 0')
// t.is(getMetricValue(lines, 'hypercore_total_blocks_uploaded'), 0, 'hypercore_total_blocks_uploaded init 0')
// t.is(getMetricValue(lines, 'hypercore_total_bytes_downloaded'), 0, 'hypercore_total_bytes_downloaded init 0')
// t.is(getMetricValue(lines, 'hypercore_total_bytes_uploaded'), 0, 'hypercore_total_bytes_uploaded init 0')

t.is(getMetricValue(lines, 'hypercore_total_wire_sync_received'), 0, 'hypercore_total_wire_sync_received init 0')
t.is(getMetricValue(lines, 'hypercore_total_wire_sync_transmitted'), 0, 'hypercore_total_wire_sync_transmitted init 0')
t.is(getMetricValue(lines, 'hypercore_total_wire_request_received'), 0, 'hypercore_total_wire_request_received init 0')
t.is(getMetricValue(lines, 'hypercore_total_wire_request_transmitted'), 0, 'hypercore_total_wire_request_transmitted init 0')
t.is(getMetricValue(lines, 'hypercore_total_wire_cancel_received'), 0, 'hypercore_total_wire_cancel_received init 0')
t.is(getMetricValue(lines, 'hypercore_total_wire_cancel_transmitted'), 0, 'hypercore_total_wire_cancel_transmitted init 0')
t.is(getMetricValue(lines, 'hypercore_total_wire_data_received'), 0, 'hypercore_total_wire_data_received init 0')
t.is(getMetricValue(lines, 'hypercore_total_wire_data_transmitted'), 0, 'hypercore_total_wire_data_transmitted init 0')
t.is(getMetricValue(lines, 'hypercore_total_wire_want_received'), 0, 'hypercore_total_wire_want_received init 0')
t.is(getMetricValue(lines, 'hypercore_total_wire_want_transmitted'), 0, 'hypercore_total_wire_want_transmitted init 0')
t.is(getMetricValue(lines, 'hypercore_total_wire_bitfield_received'), 0, 'hypercore_total_wire_bitfield_received init 0')
t.is(getMetricValue(lines, 'hypercore_total_wire_bitfield_transmitted'), 0, 'hypercore_total_wire_bitfield_transmitted init 0')
t.is(getMetricValue(lines, 'hypercore_total_wire_range_received'), 0, 'hypercore_total_wire_range_received init 0')
t.is(getMetricValue(lines, 'hypercore_total_wire_range_transmitted'), 0, 'hypercore_total_wire_range_transmitted init 0')
t.is(getMetricValue(lines, 'hypercore_total_wire_extension_received'), 0, 'hypercore_total_wire_extension_received init 0')
t.is(getMetricValue(lines, 'hypercore_total_wire_extension_transmitted'), 0, 'hypercore_total_wire_extension_transmitted init 0')
}

await core.append('block0')
Expand All @@ -46,11 +68,13 @@ test('Can register and get prometheus metrics', async (t) => {
const s2 = readCore.replicate(false)
s1.pipe(s2).pipe(s1)

await readCore.get(0)
// DEVNOTE: The precise lifecycle of when a peer is added to
// a core's replicator is complex (and there is no event for now).
// Rather than waiting for the exact events,
// we hack it out and wait a redundantly long time
await new Promise(resolve => setTimeout(resolve, 100))
// (same applies to when an update event is registered, but we could listen for that event)
await new Promise(resolve => setTimeout(resolve, 1000))

{
stats.clearCache()
Expand All @@ -62,6 +86,16 @@ test('Can register and get prometheus metrics', async (t) => {
// TODO: proper test of inflight metrics
t.is(getMetricValue(lines, 'hypercore_total_length'), 3, 'hypercore_total_length')
t.is(getMetricValue(lines, 'hypercore_total_peers'), 1, 'hypercore_total_peers')
// t.is(getMetricValue(lines, 'hypercore_total_blocks_downloaded'), 0, 'hypercore_total_blocks_downloaded')
// t.is(getMetricValue(lines, 'hypercore_total_blocks_uploaded'), 1, 'hypercore_total_blocks_uploaded')
// t.is(getMetricValue(lines, 'hypercore_total_bytes_downloaded'), 0, 'hypercore_total_bytes_downloaded')
// t.ok(getMetricValue(lines, 'hypercore_total_bytes_uploaded') > 0, 'hypercore_total_bytes_uploaded')
t.ok(getMetricValue(lines, 'hypercore_total_wire_sync_received') > 0, 'hypercore_total_wire_sync_received')
t.ok(getMetricValue(lines, 'hypercore_total_wire_sync_transmitted') > 0, 'hypercore_total_wire_sync_transmitted')
t.ok(getMetricValue(lines, 'hypercore_total_wire_want_received') > 0, 'hypercore_total_wire_want_received')
t.ok(getMetricValue(lines, 'hypercore_total_wire_data_transmitted') > 0, 'hypercore_total_wire_data_transmitted')
t.ok(getMetricValue(lines, 'hypercore_total_wire_request_received') > 0, 'hypercore_total_wire_request_received')
t.ok(getMetricValue(lines, 'hypercore_total_wire_range_transmitted') > 0, 'hypercore_total_wire_range_transmitted')
}
})

Expand Down

0 comments on commit 7f37735

Please sign in to comment.