From a9ad10cadc51f6f200dd7df19a52eab0ab076129 Mon Sep 17 00:00:00 2001 From: web3-developer <51288821+web3-developer@users.noreply.github.com> Date: Thu, 19 Sep 2024 21:38:49 +0800 Subject: [PATCH] Fluffy state network now enabled by default and improve status logs (#2640) * Enable state network by default. Create status log loop for state and beacon networks. Create status log loop for portal node. Implement stop functions. --- fluffy/conf.nim | 2 +- fluffy/network/beacon/beacon_light_client.nim | 7 +++- fluffy/network/beacon/beacon_network.nim | 17 ++++++++ fluffy/network/history/history_network.nim | 15 ++----- fluffy/network/state/state_gossip.nim | 1 - fluffy/network/state/state_network.nim | 20 +++++++++- fluffy/portal_node.nim | 40 +++++++++++++++++++ .../portal_bridge/portal_bridge_state.nim | 4 +- 8 files changed, 89 insertions(+), 17 deletions(-) diff --git a/fluffy/conf.nim b/fluffy/conf.nim index 9e3bf5294e..f2324d4038 100644 --- a/fluffy/conf.nim +++ b/fluffy/conf.nim @@ -89,7 +89,7 @@ type portalSubnetworks* {. desc: "Select which networks (Portal sub-protocols) to enable", - defaultValue: {PortalSubnetwork.history}, + defaultValue: {PortalSubnetwork.history, PortalSubnetwork.state}, name: "portal-subnetworks" .}: set[PortalSubnetwork] diff --git a/fluffy/network/beacon/beacon_light_client.nim b/fluffy/network/beacon/beacon_light_client.nim index d0b97fac1c..a806860d19 100644 --- a/fluffy/network/beacon/beacon_light_client.nim +++ b/fluffy/network/beacon/beacon_light_client.nim @@ -178,10 +178,13 @@ proc new*( ) proc start*(lightClient: LightClient) = - notice "Starting beacon light client", - trusted_block_root = lightClient.trustedBlockRoot + info "Starting beacon light client", trusted_block_root = lightClient.trustedBlockRoot lightClient.manager.start() +proc stop*(lightClient: LightClient) = + info "Stopping beacon light client" + discard lightClient.manager.stop() + proc resetToFinalizedHeader*( lightClient: LightClient, header: ForkedLightClientHeader, diff --git a/fluffy/network/beacon/beacon_network.nim b/fluffy/network/beacon/beacon_network.nim index 5ef2be4c26..544f7130b8 100644 --- a/fluffy/network/beacon/beacon_network.nim +++ b/fluffy/network/beacon/beacon_network.nim @@ -31,6 +31,7 @@ type BeaconNetwork* = ref object forkDigests*: ForkDigests trustedBlockRoot: Opt[Eth2Digest] processContentLoop: Future[void] + statusLogLoop: Future[void] func toContentIdHandler(contentKey: ContentKeyByteList): results.Opt[ContentId] = ok(toContentId(contentKey)) @@ -364,13 +365,29 @@ proc processContentLoop(n: BeaconNetwork) {.async: (raises: []).} = except CancelledError: trace "processContentLoop canceled" +proc statusLogLoop(n: BeaconNetwork) {.async: (raises: []).} = + try: + while true: + info "Beacon network status", + routingTableNodes = n.portalProtocol.routingTable.len() + + await sleepAsync(60.seconds) + except CancelledError: + trace "statusLogLoop canceled" + proc start*(n: BeaconNetwork) = info "Starting Portal beacon chain network" + n.portalProtocol.start() n.processContentLoop = processContentLoop(n) proc stop*(n: BeaconNetwork) = + info "Stopping Portal beacon chain network" + n.portalProtocol.stop() if not n.processContentLoop.isNil: n.processContentLoop.cancelSoon() + + if not n.statusLogLoop.isNil(): + n.statusLogLoop.cancelSoon() diff --git a/fluffy/network/history/history_network.nim b/fluffy/network/history/history_network.nim index 1bbb5361da..75e03e1595 100644 --- a/fluffy/network/history/history_network.nim +++ b/fluffy/network/history/history_network.nim @@ -704,17 +704,7 @@ proc processContentLoop(n: HistoryNetwork) {.async: (raises: []).} = proc statusLogLoop(n: HistoryNetwork) {.async: (raises: []).} = try: while true: - # This is the data radius percentage compared to full storage. This will - # drop a lot when using the logbase2 scale, namely `/ 2` per 1 logaritmic - # radius drop. - # TODO: Get some float precision calculus? - let radiusPercentage = - n.portalProtocol.dataRadius() div (UInt256.high() div u256(100)) - info "History network status", - radiusPercentage = radiusPercentage.toString(10) & "%", - radius = n.portalProtocol.dataRadius().toHex(), - dbSize = $(n.contentDB.size() div 1000) & "kb", routingTableNodes = n.portalProtocol.routingTable.len() await sleepAsync(60.seconds) @@ -725,6 +715,7 @@ proc start*(n: HistoryNetwork) = info "Starting Portal execution history network", protocolId = n.portalProtocol.protocolId, accumulatorRoot = hash_tree_root(n.accumulator) + n.portalProtocol.start() n.processContentLoop = processContentLoop(n) @@ -732,10 +723,12 @@ proc start*(n: HistoryNetwork) = pruneDeprecatedAccumulatorRecords(n.accumulator, n.contentDB) proc stop*(n: HistoryNetwork) = + info "Stopping Portal execution history network" + n.portalProtocol.stop() if not n.processContentLoop.isNil: n.processContentLoop.cancelSoon() - if not n.processContentLoop.isNil: + if not n.statusLogLoop.isNil: n.statusLogLoop.cancelSoon() diff --git a/fluffy/network/state/state_gossip.nim b/fluffy/network/state/state_gossip.nim index 3ff2e647c5..ec5194e4dc 100644 --- a/fluffy/network/state/state_gossip.nim +++ b/fluffy/network/state/state_gossip.nim @@ -128,7 +128,6 @@ proc gossipOffer*( debug "Offered content gossipped successfully with peers", keyBytes, peers # Currently only used for testing to gossip an entire account trie proof -# This may also be useful for the state network bridge proc recursiveGossipOffer*( p: PortalProtocol, srcNodeId: Opt[NodeId], diff --git a/fluffy/network/state/state_network.nim b/fluffy/network/state/state_network.nim index 0d54d4622d..8b1c6e67c7 100644 --- a/fluffy/network/state/state_network.nim +++ b/fluffy/network/state/state_network.nim @@ -31,6 +31,7 @@ type StateNetwork* = ref object contentDB*: ContentDB contentQueue*: AsyncQueue[(Opt[NodeId], ContentKeysList, seq[seq[byte]])] processContentLoop: Future[void] + statusLogLoop: Future[void] historyNetwork: Opt[HistoryNetwork] validateStateIsCanonical: bool @@ -221,15 +222,32 @@ proc processContentLoop(n: StateNetwork) {.async: (raises: []).} = except CancelledError: trace "processContentLoop canceled" +proc statusLogLoop(n: StateNetwork) {.async: (raises: []).} = + try: + while true: + info "State network status", + routingTableNodes = n.portalProtocol.routingTable.len() + + await sleepAsync(60.seconds) + except CancelledError: + trace "statusLogLoop canceled" + proc start*(n: StateNetwork) = info "Starting Portal execution state network", protocolId = n.portalProtocol.protocolId + n.portalProtocol.start() n.processContentLoop = processContentLoop(n) + n.statusLogLoop = statusLogLoop(n) proc stop*(n: StateNetwork) = + info "Stopping Portal execution state network" + n.portalProtocol.stop() - if not n.processContentLoop.isNil: + if not n.processContentLoop.isNil(): n.processContentLoop.cancelSoon() + + if not n.statusLogLoop.isNil(): + n.statusLogLoop.cancelSoon() diff --git a/fluffy/portal_node.nim b/fluffy/portal_node.nim index e2dce3aad0..ccb1d8151c 100644 --- a/fluffy/portal_node.nim +++ b/fluffy/portal_node.nim @@ -9,6 +9,7 @@ import results, + chronos, eth/p2p/discoveryv5/protocol, beacon_chain/spec/forks, ./network_metadata, @@ -39,6 +40,7 @@ type historyNetwork*: Opt[HistoryNetwork] stateNetwork*: Opt[StateNetwork] beaconLightClient*: Opt[LightClient] + statusLogLoop: Future[void] # Beacon light client application callbacks triggered when new finalized header # or optimistic header is available. @@ -179,7 +181,27 @@ proc new*( beaconLightClient: beaconLightClient, ) +proc statusLogLoop(n: PortalNode) {.async: (raises: []).} = + try: + while true: + # This is the data radius percentage compared to full storage. This will + # drop a lot when using the logbase2 scale, namely `/ 2` per 1 logaritmic + # radius drop. + # TODO: Get some float precision calculus? + let radiusPercentage = n.contentDB.dataRadius div (UInt256.high() div u256(100)) + + info "Portal node status", + radiusPercentage = radiusPercentage.toString(10) & "%", + radius = n.contentDB.dataRadius.toHex(), + dbSize = $(n.contentDB.size() div 1000) & "kb" + + await sleepAsync(60.seconds) + except CancelledError: + trace "statusLogLoop canceled" + proc start*(n: PortalNode) = + debug "Starting Portal node" + if n.beaconNetwork.isSome(): n.beaconNetwork.value.start() if n.historyNetwork.isSome(): @@ -189,3 +211,21 @@ proc start*(n: PortalNode) = if n.beaconLightClient.isSome(): n.beaconLightClient.value.start() + + n.statusLogLoop = statusLogLoop(n) + +proc stop*(n: PortalNode) = + debug "Stopping Portal node" + + if n.beaconNetwork.isSome(): + n.beaconNetwork.value.stop() + if n.historyNetwork.isSome(): + n.historyNetwork.value.stop() + if n.stateNetwork.isSome(): + n.stateNetwork.value.stop() + + if n.beaconLightClient.isSome(): + n.beaconLightClient.value.stop() + + if not n.statusLogLoop.isNil: + n.statusLogLoop.cancelSoon() diff --git a/fluffy/tools/portal_bridge/portal_bridge_state.nim b/fluffy/tools/portal_bridge/portal_bridge_state.nim index 23e75a12dc..7012270207 100644 --- a/fluffy/tools/portal_bridge/portal_bridge_state.nim +++ b/fluffy/tools/portal_bridge/portal_bridge_state.nim @@ -298,7 +298,9 @@ proc runBackfillGossipBlockOffersLoop( for k, v in offersMap: try: let numPeers = await portalClient.portal_stateGossip(k.to0xHex(), v.to0xHex()) - if numPeers == 0: + if numPeers > 0: + debug "Offer successfully gossipped to peers: ", numPeers, workerId + elif numPeers == 0: warn "Offer gossipped to no peers", workerId retryGossip = true break