From f4a96bc3f3207c9a7761e96e66993de3011594f0 Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Wed, 22 May 2019 10:13:15 +0300 Subject: [PATCH 01/18] [WIP] Restore the Lib2P2 builds and implement the latest wire spec Depends on https://github.com/status-im/nim-eth/pull/54 --- beacon_chain/beacon_node.nim | 12 +- beacon_chain/beacon_node_types.nim | 29 +- beacon_chain/conf.nim | 2 +- beacon_chain/eth2_network.nim | 47 ++- beacon_chain/libp2p_backend.nim | 471 ++++++++++------------------- beacon_chain/request_manager.nim | 2 +- beacon_chain/sync_protocol.nim | 44 ++- beacon_chain/version.nim | 3 +- nim.cfg | 2 + scripts/testnet0.env | 2 +- scripts/testnet1.env | 2 +- 11 files changed, 251 insertions(+), 365 deletions(-) diff --git a/beacon_chain/beacon_node.nim b/beacon_chain/beacon_node.nim index 3f61c53f2b..d4d564de52 100644 --- a/beacon_chain/beacon_node.nim +++ b/beacon_chain/beacon_node.nim @@ -86,13 +86,11 @@ proc saveValidatorKey(keyName, key: string, conf: BeaconNodeConf) = writeFile(outputFile, key) info "Imported validator key", file = outputFile -proc persistentNodeId*(conf: BeaconNodeConf): string = - ($ensureNetworkKeys(conf).pubKey)[0..5] - proc init*(T: type BeaconNode, conf: BeaconNodeConf): Future[BeaconNode] {.async.} = new result result.config = conf - result.nickname = if conf.nodename == "auto": persistentNodeId(conf) + result.networkIdentity = getPersistentNetIdentity(conf) + result.nickname = if conf.nodename == "auto": shortForm(result.networkIdentity) else: conf.nodename template fail(args: varargs[untyped]) = @@ -182,6 +180,7 @@ proc init*(T: type BeaconNode, conf: BeaconNodeConf): Future[BeaconNode] {.async # TODO sync is called when a remote peer is connected - is that the right # time to do so? let sync = result.network.protocolState(BeaconSync) + sync.chainId = 0 # TODO specify chainId sync.networkId = result.networkMetadata.networkId sync.node = result sync.db = result.db @@ -210,11 +209,10 @@ template withState( body proc connectToNetwork(node: BeaconNode) {.async.} = - let localKeys = ensureNetworkKeys(node.config) var bootstrapNodes = newSeq[BootstrapAddr]() for bootNode in node.networkMetadata.bootstrapNodes: - if bootNode.pubkey == localKeys.pubKey: + if bootNode.isSameNode(node.networkIdentity): node.isBootstrapNode = true else: bootstrapNodes.add bootNode @@ -278,7 +276,7 @@ proc updateHead(node: BeaconNode, slot: Slot): BlockRef = # TODO move all of this logic to BlockPool debug "Preparing for fork choice", stateRoot = shortLog(root), - connectedPeers = node.network.connectedPeers, + connectedPeers = node.network.peersCount, stateSlot = humaneSlotNum(state.slot), stateEpoch = humaneEpochNum(state.slot.slotToEpoch) diff --git a/beacon_chain/beacon_node_types.nim b/beacon_chain/beacon_node_types.nim index b7da13a9f8..0e5306d3f0 100644 --- a/beacon_chain/beacon_node_types.nim +++ b/beacon_chain/beacon_node_types.nim @@ -1,21 +1,8 @@ -import # Beacon Node - eth/[p2p, keys], - spec/[bitfield, digest], - beacon_chain_db, conf, mainchain_monitor, eth2_network, - ./time - -import # Attestation Pool +import + sets, deques, tables, + eth/keys, spec/[bitfield, datatypes, crypto, digest], - deques, tables - # block_pool - -import # Block Pool - spec/[datatypes, digest], - beacon_chain_db, - tables - -import # Validator Pool - spec/crypto, tables + beacon_chain_db, conf, mainchain_monitor, eth2_network, time type @@ -26,13 +13,13 @@ type # ############################################# BeaconNode* = ref object nickname*: string - network*: EthereumNode + network*: Eth2Node + networkIdentity*: Eth2NodeIdentity networkMetadata*: NetworkMetadata requestManager*: RequestManager isBootstrapNode*: bool db*: BeaconChainDB config*: BeaconNodeConf - keys*: KeyPair attachedValidators*: ValidatorPool blockPool*: BlockPool attestationPool*: AttestationPool @@ -258,10 +245,10 @@ type validators*: Table[ValidatorPubKey, AttachedValidator] RequestManager* = object - network*: EthereumNode + network*: Eth2Node NetworkMetadata* = object - networkId*: uint64 + networkId*: uint8 networkGeneration*: uint64 genesisRoot*: Eth2Digest bootstrapNodes*: seq[BootstrapAddr] diff --git a/beacon_chain/conf.nim b/beacon_chain/conf.nim index a0efd1066a..7e9311a4ce 100644 --- a/beacon_chain/conf.nim +++ b/beacon_chain/conf.nim @@ -79,7 +79,7 @@ type of createTestnet: networkId* {. - desc: "An unique numeric identifier for the network".}: uint64 + desc: "An unique numeric identifier for the network".}: uint8 validatorsDir* {. desc: "Directory containing validator descriptors named vXXXXXXX.deposit.json" diff --git a/beacon_chain/eth2_network.nim b/beacon_chain/eth2_network.nim index 03f1a5a4ee..29c1348801 100644 --- a/beacon_chain/eth2_network.nim +++ b/beacon_chain/eth2_network.nim @@ -1,5 +1,6 @@ import - options, chronos, json_serialization, strutils, + options, tables, + chronos, json_serialization, strutils, chronicles, spec/digest, version, conf @@ -20,6 +21,7 @@ when useRLPx: type Eth2Node* = EthereumNode + Eth2NodeIdentity* = KeyPair BootstrapAddr* = ENode template libp2pProtocol*(name, version: string) {.pragma.} @@ -59,7 +61,7 @@ when useRLPx: if extPorts.isSome: (result.tcpPort, result.udpPort) = extPorts.get() - proc ensureNetworkKeys*(conf: BeaconNodeConf): KeyPair = + proc getPersistentNetIdentity*(conf: BeaconNodeConf): Eth2NodeIdentity = let privateKeyFile = conf.dataDir / "network.privkey" var privKey: PrivateKey if not fileExists(privateKeyFile): @@ -74,10 +76,16 @@ when useRLPx: proc getPersistenBootstrapAddr*(conf: BeaconNodeConf, ip: IpAddress, port: Port): BootstrapAddr = let - keys = ensureNetworkKeys(conf) + identity = getPersistentNetIdentity(conf) address = Address(ip: ip, tcpPort: port, udpPort: port) - initENode(keys.pubKey, address) + initENode(identity.pubKey, address) + + proc isSameNode*(bootstrapNode: BootstrapAddr, id: Eth2NodeIdentity): bool = + bootstrapNode.pubKey == id.pubKey + + proc shortForm*(id: Eth2NodeIdentity): string = + ($id.pubKey)[0..5] proc writeValue*(writer: var JsonWriter, value: BootstrapAddr) {.inline.} = writer.writeValue $value @@ -87,7 +95,7 @@ when useRLPx: proc createEth2Node*(conf: BeaconNodeConf): Future[EthereumNode] {.async.} = let - keys = ensureNetworkKeys(conf) + keys = getPersistentNetIdentity(conf) (ip, tcpPort, udpPort) = setupNat(conf) address = Address(ip: ip, tcpPort: tcpPort, @@ -104,8 +112,8 @@ when useRLPx: proc init*(T: type BootstrapAddr, str: string): T = initENode(str) - func connectedPeers*(enode: EthereumNode): int = - enode.peerPool.len + func peersCount*(node: Eth2Node): int = + node.peerPool.len else: import @@ -117,6 +125,7 @@ else: type BootstrapAddr* = PeerInfo + Eth2NodeIdentity* = PeerInfo const netBackendName* = "libp2p" @@ -141,6 +150,27 @@ else: await node.init() return node + proc getPersistentNetIdentity*(conf: BeaconNodeConf): Eth2NodeIdentity = + # Using waitFor here is reasonable, because this proc is needed only + # prior to connecting to the network. The RLPx alternative reads from + # file and it's much easier to use if it's not async. + # TODO: revisit in the future when we have our own Lib2P2 implementation. + let daemon = waitFor newDaemonApi() + result = waitFor daemon.identity() + waitFor daemon.close() + + proc getPersistenBootstrapAddr*(conf: BeaconNodeConf, + ip: IpAddress, port: Port): BootstrapAddr = + # TODO what about the ports? + getPersistentNetIdentity(conf) + + proc isSameNode*(bootstrapNode: BootstrapAddr, id: Eth2NodeIdentity): bool = + bootstrapNode == id + + proc shortForm*(id: Eth2NodeIdentity): string = + # TODO: Make this shorter + $id + proc connectToNetwork*(node: Eth2Node, bootstrapNodes: seq[PeerInfo]) {.async.} = # TODO: perhaps we should do these in parallel for bootstrapNode in bootstrapNodes: @@ -158,3 +188,6 @@ else: proc loadConnectionAddressFile*(filename: string): PeerInfo = Json.loadFile(filename, PeerInfo) + func peersCount*(node: Eth2Node): int = + node.peers.len + diff --git a/beacon_chain/libp2p_backend.nim b/beacon_chain/libp2p_backend.nim index 957d202fbc..c4a908079c 100644 --- a/beacon_chain/libp2p_backend.nim +++ b/beacon_chain/libp2p_backend.nim @@ -1,11 +1,12 @@ import - options, macros, algorithm, + options, macros, algorithm, random, tables, std_shims/[macros_shim, tables_shims], chronos, chronicles, libp2p/daemon/daemonapi, faststreams/output_stream, serialization, + eth/async_utils, eth/p2p/p2p_protocol_dsl, ssz export - daemonapi + daemonapi, p2pProtocol type Eth2Node* = ref object of RootObj @@ -14,9 +15,9 @@ type protocolStates*: seq[RootRef] Peer* = ref object - network: Eth2Node - id: PeerID - connectionState: ConnectionState + network*: Eth2Node + id*: PeerID + connectionState*: ConnectionState awaitedMessages: Table[CompressedMsgId, FutureBase] protocolStates*: seq[RootRef] @@ -48,11 +49,6 @@ type CompressedMsgId = tuple protocolIndex, msgId: int - MessageKind* = enum - msgNotification, - msgRequest, - msgResponse - PeerStateInitializer* = proc(peer: Peer): RootRef {.gcsafe.} NetworkStateInitializer* = proc(network: EthereumNode): RootRef {.gcsafe.} HandshakeStep* = proc(peer: Peer, handshakeStream: P2PStream): Future[void] {.gcsafe.} @@ -94,7 +90,10 @@ var # Nim to not consider them GcSafe violations: template allProtocols: auto = {.gcsafe.}: gProtocols -proc disconnect*(peer: Peer) {.async.} = +proc `$`*(peer: Peer): string = $peer.id + +proc disconnect*(peer: Peer, reason: DisconnectionReason, notifyOtherPeer = false) {.async.} = + # TODO: How should we notify the other peer? if peer.connectionState notin {Disconnecting, Disconnected}: peer.connectionState = Disconnecting await peer.network.daemon.disconnect(peer.id) @@ -110,7 +109,7 @@ proc disconnectAndRaise(peer: Peer, reason: DisconnectionReason, msg: string) {.async.} = let r = reason - await peer.disconnect() + await peer.disconnect(reason) raisePeerDisconnected(msg, reason) proc init*(node: Eth2Node) {.async.} = @@ -130,18 +129,14 @@ proc init*(node: Eth2Node) {.async.} = include eth/p2p/p2p_backends_helpers include eth/p2p/p2p_tracing -import typetraits - proc readMsg(stream: P2PStream, MsgType: type, - timeout = 10000): Future[Option[MsgType]] {.async.} = + timeout = 10.seconds): Future[Option[MsgType]] {.async.} = var timeout = sleepAsync timeout var sizePrefix: uint32 var readSizePrefix = stream.transp.readExactly(addr sizePrefix, sizeof(sizePrefix)) await readSizePrefix or timeout if not readSizePrefix.finished: return - debug "EXPECTING MSG", msg = MsgType.name, size = sizePrefix.int - var msgBytes = newSeq[byte](sizePrefix.int + sizeof(sizePrefix)) copyMem(addr msgBytes[0], addr sizePrefix, sizeof(sizePrefix)) var readBody = stream.transp.readExactly(addr msgBytes[sizeof(sizePrefix)], sizePrefix.int) @@ -166,50 +161,55 @@ proc sendBytes(stream: P2PStream, bytes: Bytes) {.async.} = proc makeEth2Request(peer: Peer, protocolId: string, requestBytes: Bytes, ResponseMsg: type, - timeout = 10000): Future[Option[ResponseMsg]] {.async.} = + timeout = 10.seconds): Future[Option[ResponseMsg]] {.async.} = var stream = await peer.network.daemon.openStream(peer.id, @[protocolId]) # TODO how does openStream fail? Set a timeout here and handle it let sent = await stream.transp.write(requestBytes) # TODO: Should I check that `sent` is equal to the desired number of bytes return await stream.readMsg(ResponseMsg, timeout) -proc handshakeImpl(peer: Peer, - handshakeSendFut: Future[void], - handshakeStream: P2PStream, - timeout: int, - HandshakeType: type): Future[HandshakeType] {.async.} = - await handshakeSendFut - let response = await handshakeStream.readMsg(HandshakeType, timeout) - if response.isSome: - return response.get - else: - await peer.disconnectAndRaise(BreachOfProtocol, "Handshake not completed in time") - proc p2pStreamName(MsgType: type): string = mixin msgProtocol, protocolInfo, msgId MsgType.msgProtocol.protocolInfo.messages[MsgType.msgId].libp2pProtocol -macro handshake*(peer: Peer, timeout = 10000, sendCall: untyped): untyped = +template handshakeImpl*(HandshakeTypeExpr: untyped, + # TODO: we cannot use a type parameter above + # because of the following Nim issue: + # + peerExpr: Peer, + streamExpr: P2PStream, + lazySendCall: Future[void], + timeoutExpr: Duration): auto = + # We make sure the inputs are evaluated only once. let - msgName = $sendCall[0] - msgType = newDotExpr(ident"CurrentProtocol", ident(msgName)) - handshakeStream = ident "handshakeStream" - handshakeImpl = bindSym "handshakeImpl" - await = ident "await" - - sendCall.insert(1, handshakeStream) - - result = quote do: - proc payload(peer: Peer, `handshakeStream`: P2PStream): Future[`msgType`] {.async.} = - var `handshakeStream` = `handshakeStream` - if `handshakeStream` == nil: - `handshakeStream` = `await` openStream(peer.network.daemon, - peer.id, - @[p2pStreamName(`msgType`)], - `timeout`) - return `await` `handshakeImpl`(peer, `sendCall`, `handshakeStream`, `timeout`, `msgType`) + stream = streamExpr + peer = peerExpr + timeout = timeoutExpr + + # TODO: This is a work-around for a Nim issue. Please note that it's + # semantically wrong, so if you get a compilation failure, try to + # remove it (perhaps Nim got fixed) + type HandshakeType = type(HandshakeTypeExpr) + + proc asyncStep(stream: P2PStream): Future[HandshakeType] {.async.} = + var stream = stream + if stream == nil: + stream = await openStream(peer.network.daemon, peer.id, + @[p2pStreamName(HandshakeType)], + # TODO openStream should accept Duration + int milliseconds(timeout)) + + # Please pay attention that `lazySendCall` is evaluated lazily here. + # For this reason `handshakeImpl` must remain a template. + await lazySendCall + + let response = await readMsg(stream, HandshakeType, timeout) + if response.isSome: + return response.get + else: + await disconnectAndRaise(peer, BreachOfProtocol, "Handshake not completed in time") - payload(`peer`, `handshakeStream`) + asyncStep(stream) proc getCompressedMsgId(MsgType: type): CompressedMsgId = mixin msgProtocol, protocolInfo, msgId @@ -312,8 +312,6 @@ proc registerProtocol(protocol: ProtocolInfo) = for i in 0 ..< gProtocols.len: gProtocols[i].index = i -template libp2pProtocol*(name, version: string) {.pragma.} - proc getRequestProtoName(fn: NimNode): NimNode = # `getCustomPragmaVal` doesn't work yet on regular nnkProcDef nodes # (TODO: file as an issue) @@ -326,34 +324,14 @@ proc getRequestProtoName(fn: NimNode): NimNode = error "All stream opening procs must have the 'libp2pProtocol' pragma specified.", fn -macro p2pProtocolImpl(name: static[string], - version: static[uint], - body: untyped, - timeout: static[int] = defaultOutgoingReqTimeout, - shortName: static[string] = "", - peerState = type(nil), - networkState = type(nil)): untyped = - ## The macro used to defined P2P sub-protocols. See README. +template libp2pProtocol*(name, version: string) {.pragma.} + +proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = var - # XXX: deal with a Nim bug causing the macro params to be - # zero when they are captured by a closure: - defaultTimeout = timeout - protoName = name - nextId = -1 - protoNameIdent = ident(protoName) - outTypes = newNimNode(nnkStmtList) - outSendProcs = newNimNode(nnkStmtList) - outRecvProcs = newNimNode(nnkStmtList) - outProcRegistrations = newNimNode(nnkStmtList) response = ident"response" name_openStream = newTree(nnkPostfix, ident("*"), ident"openStream") outputStream = ident"outputStream" currentProtocolSym = ident"CurrentProtocol" - protocol = ident(protoName & "Protocol") - peerState = verifyStateType peerState.getType - networkState = verifyStateType networkState.getType - handshake = newNilLit() - disconnectHandler = newNilLit() Format = ident"SSZ" Option = bindSym "Option" UntypedResponse = bindSym "UntypedResponse" @@ -365,98 +343,57 @@ macro p2pProtocolImpl(name: static[string], Int = ident "int" Void = ident "void" Peer = bindSym "Peer" + Eth2Node = bindSym "Eth2Node" writeField = bindSym "writeField" - createNetworkState = bindSym "createNetworkState" - createPeerState = bindSym "createPeerState" getOutput = bindSym "getOutput" messagePrinter = bindSym "messagePrinter" - initProtocol = bindSym "initProtocol" getRecipient = bindSym "getRecipient" peerFromStream = bindSym "peerFromStream" makeEth2Request = bindSym "makeEth2Request" sendMsg = bindSym "sendMsg" sendBytes = bindSym "sendBytes" - getState = bindSym "getState" - getNetworkState = bindSym "getNetworkState" resolveNextMsgFutures = bindSym "resolveNextMsgFutures" + milliseconds = bindSym "milliseconds" + registerMsg = bindSym "registerMsg" + initProtocol = bindSym "initProtocol" + bindSymOp = bindSym "bindSym" + msgRecipient = ident"msgRecipient" + sendTo = ident"sendTo" + writer = ident"writer" + recordStartMemo = ident"recordStartMemo" + receivedMsg = ident"msg" + daemon = ident "daemon" + stream = ident "stream" + await = ident "await" + peerIdent = ident "peer" - proc augmentUserHandler(userHandlerProc: NimNode, - msgKind = msgNotification, - extraDefinitions: NimNode = nil) = - ## Turns a regular proc definition into an async proc and adds - ## the helpers for accessing the peer and network protocol states. - - userHandlerProc.addPragma ident"gcsafe" - userHandlerProc.addPragma ident"async" - - # We allow the user handler to use `openarray` params, but we turn - # those into sequences to make the `async` pragma happy. - for i in 1 ..< userHandlerProc.params.len: - var param = userHandlerProc.params[i] - param[^2] = chooseFieldType(param[^2]) - - var userHandlerDefinitions = newStmtList() - - userHandlerDefinitions.add quote do: - type `currentProtocolSym` = `protoNameIdent` - - if extraDefinitions != nil: - userHandlerDefinitions.add extraDefinitions - - # Define local accessors for the peer and the network protocol states - # inside each user message handler proc (e.g. peer.state.foo = bar) - if peerState != nil: - userHandlerDefinitions.add quote do: - template state(p: `Peer`): `peerState` = - cast[`peerState`](`getState`(p, `protocol`)) - - if networkState != nil: - userHandlerDefinitions.add quote do: - template networkState(p: `Peer`): `networkState` = - cast[`networkState`](`getNetworkState`(p.network, `protocol`)) - - userHandlerProc.body.insert 0, userHandlerDefinitions - - proc liftEventHandler(doBlock: NimNode, handlerName: string): NimNode = - ## Turns a "named" do block to a regular async proc - ## (e.g. onPeerConnected do ...) - result = newTree(nnkProcDef) - doBlock.copyChildrenTo(result) - result.name = genSym(nskProc, protoName & handlerName) - augmentUserHandler result - outRecvProcs.add result + new result - proc addMsgHandler(n: NimNode, msgKind = msgNotification, - responseRecord: NimNode = nil): NimNode = - if n[0].kind == nnkPostfix: - macros.error("p2pProcotol procs are public by default. " & - "Please remove the postfix `*`.", n) + result.PeerType = Peer + result.NetworkType = Eth2Node + result.registerProtocol = bindSym "registerProtocol" + result.setEventHandlers = bindSym "setEventHandlers" - inc nextId + result.afterProtocolInit = proc (p: P2PProtocol) = + p.onPeerConnected.params.add newIdentDefs(ident"handshakeStream", P2PStream) + result.implementMsg = proc (p: P2PProtocol, msg: Message, resp: Message = nil) = let + n = msg.procDef + msgId = newLit(msg.id) msgIdent = n.name - msgName = $n.name - - var + msgName = $msgIdent + msgKind = msg.kind + msgRecName = msg.recIdent + responseRecord = if resp != nil: resp.recIdent else: nil userPragmas = n.pragma + var # variables used in the sending procs - msgRecipient = ident"msgRecipient" - sendTo = ident"sendTo" - writer = ident"writer" - recordStartMemo = ident"recordStartMemo" - reqTimeout: NimNode appendParams = newNimNode(nnkStmtList) paramsToWrite = newSeq[NimNode](0) - msgId = newLit(nextId) # variables used in the receiving procs - receivedMsg = ident"msg" - daemon = ident "daemon" - stream = ident "stream" - await = ident "await" - peerIdent = ident "peer" tracing = newNimNode(nnkStmtList) # nodes to store the user-supplied message handling proc if present @@ -464,36 +401,10 @@ macro p2pProtocolImpl(name: static[string], userHandlerCall: NimNode = nil awaitUserHandler = newStmtList() - # a record type associated with the message - msgRecord = newIdentNode(msgName & "Obj") - msgRecordFields = newTree(nnkRecList) - msgRecordBody = newTree(nnkObjectTy, - newEmptyNode(), - newEmptyNode(), - msgRecordFields) - - result = msgRecord - - if msgKind == msgRequest: - # If the request proc has a default timeout specified, remove it from - # the signature for now so we can generate the `thunk` proc without it. - # The parameter will be added back later only for to the sender proc. - # When the timeout is not specified, we use a default one. - reqTimeout = popTimeoutParam(n) - if reqTimeout == nil: - reqTimeout = newTree(nnkIdentDefs, - ident"timeout", - Int, newLit(defaultTimeout)) - if n.body.kind != nnkEmpty: - # Implement the receiving thunk proc that deserialzed the - # message parameters and calls the user proc: - userHandlerProc = n.copyNimTree - userHandlerProc.name = genSym(nskProc, msgName) - # This is the call to the user supplied handler. # Here we add only the initial params, the rest will be added later. - userHandlerCall = newCall(userHandlerProc.name) + userHandlerCall = newCall(msg.userHandler.name) # When there is a user handler, it must be awaited in the thunk proc. # Above, by default `awaitUserHandler` is set to a no-op statement list. awaitUserHandler = newCall(await, userHandlerCall) @@ -502,9 +413,9 @@ macro p2pProtocolImpl(name: static[string], if msgKind == msgRequest: # Request procs need an extra param - the stream where the response # should be written: - userHandlerProc.params.insert(1, newIdentDefs(stream, P2PStream)) + msg.userHandler.params.insert(1, newIdentDefs(stream, P2PStream)) userHandlerCall.add stream - let peer = userHandlerProc.params[2][0] + let peer = msg.userHandler.params[2][0] extraDefs = quote do: # Jump through some hoops to work aroung # https://github.com/nim-lang/Nim/issues/6248 @@ -514,24 +425,18 @@ macro p2pProtocolImpl(name: static[string], # Resolve the Eth2Peer from the LibP2P data received in the thunk userHandlerCall.add peerIdent - augmentUserHandler userHandlerProc, msgKind, extraDefs - outRecvProcs.add userHandlerProc + msg.userHandler.addPreludeDefs extraDefs + p.outRecvProcs.add msg.userHandler elif msgName == "status": - awaitUserHandler = quote do: - `await` `handshake`(`peerIdent`, `stream`) + #awaitUserHandler = quote do: + # `await` `handshake`(`peerIdent`, `stream`) + discard + # TODO: revisit this for param, paramType in n.typedParams(skip = 1): paramsToWrite.add param - # Each message has a corresponding record type. - # Here, we create its fields one by one: - msgRecordFields.add newTree(nnkIdentDefs, - newTree(nnkPostfix, ident("*"), param), # The fields are public - chooseFieldType(paramType), # some types such as openarray - # are automatically remapped - newEmptyNode()) - # If there is user message handler, we'll place a call to it by # unpacking the fields of the received message: if userHandlerCall != nil: @@ -541,12 +446,11 @@ macro p2pProtocolImpl(name: static[string], tracing = quote do: logReceivedMsg(`stream`.peer, `receivedMsg`.get) - let requestDataTimeout = newLit(defaultIncomingReqTimeout) - + let requestDataTimeout = newCall(milliseconds, newLit(defaultIncomingReqTimeout)) let thunkName = ident(msgName & "_thunk") var thunkProc = quote do: proc `thunkName`(`daemon`: `DaemonAPI`, `stream`: `P2PStream`) {.async, gcsafe.} = - var `receivedMsg` = `await` readMsg(`stream`, `msgRecord`, `requestDataTimeout`) + var `receivedMsg` = `await` readMsg(`stream`, `msgRecName`, `requestDataTimeout`) if `receivedMsg`.isNone: # TODO: This peer is misbehaving, perhaps we should penalize him somehow return @@ -558,59 +462,76 @@ macro p2pProtocolImpl(name: static[string], for p in userPragmas: thunkProc.addPragma p - outRecvProcs.add thunkProc - - outTypes.add quote do: - # This is a type featuring a single field for each message param: - type `msgRecord`* = `msgRecordBody` - - # Add a helper template for accessing the message type: - # e.g. p2p.hello: - template `msgIdent`*(T: type `protoNameIdent`): type = `msgRecord` - template msgId*(T: type `msgRecord`): int = `msgId` - template msgProtocol*(T: type `msgRecord`): type = `protoNameIdent` + p.outRecvProcs.add thunkProc var msgSendProc = n let msgSendProcName = n.name - outSendProcs.add msgSendProc + p.outSendProcs.add msgSendProc # TODO: check that the first param has the correct type msgSendProc.params[1][0] = sendTo - if nextId == 0: msgSendProc.params[1][1] = P2PStream msgSendProc.addPragma ident"gcsafe" # Add a timeout parameter for all request procs case msgKind of msgRequest: - msgSendProc.params.add reqTimeout + # Add a timeout parameter for all request procs + msgSendProc.params.add msg.timeoutParam of msgResponse: # A response proc must be called with a response object that originates # from a certain request. Here we change the Peer parameter at position # 1 to the correct strongly-typed ResponseType. The incoming procs still # gets the normal Peer paramter. - let ResponseType = newTree(nnkBracketExpr, Response, msgRecord) + let ResponseType = newTree(nnkBracketExpr, Response, msgRecName) msgSendProc.params[1][1] = ResponseType - outSendProcs.add quote do: + p.outSendProcs.add quote do: template send*(r: `ResponseType`, args: varargs[untyped]): auto = `msgSendProcName`(r, args) else: discard # We change the return type of the sending proc to a Future. # If this is a request proc, the future will return the response record. - let rt = case msgKind - of msgRequest: newTree(nnkBracketExpr, Option, responseRecord) - of msgResponse, msgNotification: Void + let rt = if msgKind != msgRequest: Void + else: newTree(nnkBracketExpr, Option, responseRecord) msgSendProc.params[0] = newTree(nnkBracketExpr, ident("Future"), rt) - let msgBytes = ident"msgBytes" - - # Make the send proc public - msgSendProc.name = newTree(nnkPostfix, ident("*"), msgSendProc.name) + if msgKind == msgHandshake: + var + rawSendProc = msgName & "RawSend" + handshakeTypeName = $msgRecName + handshakeExchanger = msg.createSendProc(nnkMacroDef) + paramsArray = newTree(nnkBracket).appendAllParams(handshakeExchanger) + bindSym = ident "bindSym" + getAst = ident "getAst" + handshakeImpl = ident "handshakeImpl" + + # TODO: macros.body triggers an assertion error when the proc type is nnkMacroDef + handshakeExchanger[6] = quote do: + let + stream = ident"handshakeStream" + rawSendProc = `bindSymOp` `rawSendProc` + params = `paramsArray` + lazySendCall = newCall(rawSendProc, params) + peer = params[0] + timeout = params[^1] + + lazySendCall[1] = stream + lazySendCall.del(lazySendCall.len - 1) + + return `getAst`(`handshakeImpl`(`msgRecName`, peer, stream, lazySendCall, timeout)) + + p.outSendProcs.add handshakeExchanger + + msgSendProc.params[1][1] = P2PStream + msgSendProc.name = ident rawSendProc + else: + # Make the send proc public + msgSendProc.name = msg.identWithExportMarker let initWriter = quote do: var `outputStream` = init OutputStream var `writer` = init(WriterType(`Format`), `outputStream`) - var `recordStartMemo` = beginRecord(`writer`, `msgRecord`) + var `recordStartMemo` = beginRecord(`writer`, `msgRecName`) for param in paramsToWrite: appendParams.add newCall(writeField, writer, newLit($param), param) @@ -618,6 +539,7 @@ macro p2pProtocolImpl(name: static[string], when tracingEnabled: appendParams.add logSentMsgFields(msgRecipient, protocol, msgName, paramsToWrite) + let msgBytes = ident"msgBytes" let finalizeRequest = quote do: endRecord(`writer`, `recordStartMemo`) let `msgBytes` = `getOutput`(`outputStream`) @@ -630,13 +552,13 @@ macro p2pProtocolImpl(name: static[string], when false: var openStreamProc = n.copyNimTree var openStreamProc.name = name_openStream - openStreamProc.params.insert 1, newIdentDefs(ident"T", msgRecord) + openStreamProc.params.insert 1, newIdentDefs(ident"T", msgRecName) if msgKind == msgRequest: - let timeout = reqTimeout[0] + let timeout = msg.timeoutParam[0] quote: `makeEth2Request`(`msgRecipient`, `msgProto`, `msgBytes`, `responseRecord`, `timeout`) - elif nextId == 0: + elif msgId.intVal == 0: quote: `sendBytes`(`sendTo`, `msgBytes`) else: quote: `sendMsg`(`msgRecipient`, `msgProto`, `msgBytes`) @@ -650,109 +572,16 @@ macro p2pProtocolImpl(name: static[string], `finalizeRequest` return `sendCall` - outProcRegistrations.add( - newCall(bindSym("registerMsg"), - protocol, + p.outProcRegistrations.add( + newCall(registerMsg, + p.protocolInfoVar, newLit(msgName), thunkName, msgProto, - newTree(nnkBracketExpr, messagePrinter, msgRecord))) - - outTypes.add quote do: - # Create a type acting as a pseudo-object representing the protocol - # (e.g. p2p) - type `protoNameIdent`* = object - - if peerState != nil: - outTypes.add quote do: - template State*(P: type `protoNameIdent`): type = `peerState` - - if networkState != nil: - outTypes.add quote do: - template NetworkState*(P: type `protoNameIdent`): type = `networkState` - - for n in body: - case n.kind - of {nnkCall, nnkCommand}: - if eqIdent(n[0], "nextID"): - discard - elif eqIdent(n[0], "requestResponse"): - # `requestResponse` can be given a block of 2 or more procs. - # The last one is considered to be a response message, while - # all preceeding ones are requests triggering the response. - # The system makes sure to automatically insert a hidden `reqId` - # parameter used to discriminate the individual messages. - block processReqResp: - if n.len == 2 and n[1].kind == nnkStmtList: - var procs = newSeq[NimNode](0) - for def in n[1]: - if def.kind == nnkProcDef: - procs.add(def) - if procs.len > 1: - let responseRecord = addMsgHandler(procs[^1], - msgKind = msgResponse) - for i in 0 .. procs.len - 2: - discard addMsgHandler(procs[i], - msgKind = msgRequest, - responseRecord = responseRecord) - - # we got all the way to here, so everything is fine. - # break the block so it doesn't reach the error call below - break processReqResp - macros.error("requestResponse expects a block with at least two proc definitions") - elif eqIdent(n[0], "onPeerConnected"): - var handshakeProc = liftEventHandler(n[1], "Handshake") - handshakeProc.params.add newIdentDefs(ident"handshakeStream", P2PStream) - handshake = handshakeProc.name - elif eqIdent(n[0], "onPeerDisconnected"): - disconnectHandler = liftEventHandler(n[1], "PeerDisconnect").name - else: - macros.error(repr(n) & " is not a recognized call in P2P protocol definitions", n) - of nnkProcDef: - discard addMsgHandler(n) - - of nnkCommentStmt: - discard - - else: - macros.error("illegal syntax in a P2P protocol definition", n) - - let peerInit = if peerState == nil: newNilLit() - else: newTree(nnkBracketExpr, createPeerState, peerState) - - let netInit = if networkState == nil: newNilLit() - else: newTree(nnkBracketExpr, createNetworkState, networkState) - - result = newNimNode(nnkStmtList) - result.add outTypes - result.add quote do: - # One global variable per protocol holds the protocol run-time data - var p = `initProtocol`(`protoName`, `peerInit`, `netInit`) - var `protocol` = addr p - - # The protocol run-time data is available as a pseudo-field - # (e.g. `p2p.protocolInfo`) - template protocolInfo*(P: type `protoNameIdent`): ProtocolInfo = `protocol` - - result.add outSendProcs, outRecvProcs, outProcRegistrations - result.add quote do: - setEventHandlers(`protocol`, `handshake`, `disconnectHandler`) - - result.add newCall(bindSym("registerProtocol"), protocol) - - when defined(debugP2pProtocol) or defined(debugMacros): - echo repr(result) + newTree(nnkBracketExpr, messagePrinter, msgRecName))) -macro p2pProtocol*(protocolOptions: untyped, body: untyped): untyped = - let protoName = $(protocolOptions[0]) - result = protocolOptions - result[0] = bindSym"p2pProtocolImpl" - result.add(newTree(nnkExprEqExpr, - ident("name"), - newLit(protoName))) - result.add(newTree(nnkExprEqExpr, - ident("body"), - body)) + result.implementProtocolInit = proc (p: P2PProtocol): NimNode = + return newCall(initProtocol, newLit(p.name), p.peerInit, p.netInit) proc makeMessageHandler[MsgType](msgHandler: proc(msg: MsgType)): P2PPubSubCallback = result = proc(api: DaemonAPI, ticket: PubsubTicket, msg: PubSubMessage): Future[bool] {.async.} = @@ -764,6 +593,16 @@ proc subscribe*[MsgType](node: EthereumNode, msgHandler: proc(msg: MsgType)) {.async.} = discard await node.daemon.pubsubSubscribe(topic, makeMessageHandler(msgHandler)) -proc broadcast*(node: Eth2Node, topic: string, msg: auto) {.async.} = - await node.daemon.pubsubPublish(topic, SSZ.encode(msg)) +proc broadcast*(node: Eth2Node, topic: string, msg: auto) = + traceAsyncErrors node.daemon.pubsubPublish(topic, SSZ.encode(msg)) + +# TODO: +# At the moment, this is just a compatiblity shim for the existing RLPx functionality. +# The filtering is not implemented properly yet. +iterator randomPeers*(node: EthereumNode, maxPeers: int, Protocol: type): Peer = + var peers = newSeq[Peer]() + for _, peer in pairs(node.peers): peers.add peer + shuffle peers + if peers.len > maxPeers: peers.setLen(maxPeers) + for p in peers: yield p diff --git a/beacon_chain/request_manager.nim b/beacon_chain/request_manager.nim index c4ffa91200..8c12044e2e 100644 --- a/beacon_chain/request_manager.nim +++ b/beacon_chain/request_manager.nim @@ -5,7 +5,7 @@ import eth2_network, beacon_node_types, sync_protocol, eth/async_utils -proc init*(T: type RequestManager, network: EthereumNode): T = +proc init*(T: type RequestManager, network: Eth2Node): T = T(network: network) type diff --git a/beacon_chain/sync_protocol.nim b/beacon_chain/sync_protocol.nim index 1beea1da78..65613b7df3 100644 --- a/beacon_chain/sync_protocol.nim +++ b/beacon_chain/sync_protocol.nim @@ -1,5 +1,5 @@ import - options, tables, sequtils, algorithm, + options, tables, sequtils, algorithm, sets, macros, chronicles, chronos, ranges/bitranges, spec/[datatypes, crypto, digest, helpers], eth/rlp, beacon_node_types, eth2_network, beacon_chain_db, block_pool, time, ssz @@ -19,7 +19,8 @@ type ValidatorSet = seq[Validator] BeaconSyncState* = ref object - networkId*: uint64 + networkId*: uint8 + chainId*: uint64 node*: BeaconNode db*: BeaconChainDB @@ -78,6 +79,7 @@ p2pProtocol BeaconSync(version = 1, protocolVersion = 1 # TODO: Spec doesn't specify this yet node = peer.networkState.node networkId = peer.networkState.networkId + chainId = peer.networkState.networkId blockPool = node.blockPool finalizedHead = blockPool.finalizedHead headBlock = blockPool.head.blck @@ -85,9 +87,9 @@ p2pProtocol BeaconSync(version = 1, bestSlot = headBlock.slot latestFinalizedEpoch = finalizedHead.slot.slot_to_epoch() - let m = await handshake(peer, timeout = 10.seconds, - status(networkId, finalizedHead.blck.root, - latestFinalizedEpoch, bestRoot, bestSlot)) + let m = await peer.hello(networkId, chainId, finalizedHead.blck.root, + latestFinalizedEpoch, bestRoot, bestSlot, + timeout = 10.seconds) if m.networkId != networkId: await peer.disconnect(UselessPeer) @@ -130,16 +132,38 @@ p2pProtocol BeaconSync(version = 1, except CatchableError: warn "Failed to sync with peer", peer, err = getCurrentExceptionMsg() - proc status( + handshake: + proc hello( peer: Peer, - networkId: uint64, + networkId: uint8, + chainId: uint64, latestFinalizedRoot: Eth2Digest, latestFinalizedEpoch: Epoch, bestRoot: Eth2Digest, bestSlot: Slot) {.libp2pProtocol("hello", "1.0.0").} + proc sendGoodbye(peer: Peer, reason: DisconnectionReason) + + requestResponse: + proc getStatus( + peer: Peer, + sha: Eth2Digest, + userAgent: string, + timestamp: uint64) = + + # TODO: How should this be implemented? + # https://github.com/ethereum/eth2.0-specs/blob/dev/specs/networking/rpc-interface.md#get-status + await response.send(sha, userAgent, timestamp) + + proc status(peer: Peer, sha: Eth2Digest, userAgent: string, timestamp: uint64) + + nextId 10 + requestResponse: - proc getBeaconBlockRoots(peer: Peer, fromSlot: Slot, maxRoots: int) = + proc getBeaconBlockRoots( + peer: Peer, + fromSlot: Slot, + maxRoots: int) {.libp2pProtocol("rpc/beacon_block_roots", "1.0.0").} = let maxRoots = min(MaxRootsToRequest, maxRoots) var s = fromSlot var roots = newSeqOfCap[(Eth2Digest, Slot)](maxRoots) @@ -206,10 +230,12 @@ p2pProtocol BeaconSync(version = 1, proc beaconBlockHeaders(peer: Peer, blockHeaders: openarray[BeaconBlockHeader]) + # TODO move this at the bottom, because it's not in the spec yet, but it will + # consume a `method_id` requestResponse: proc getAncestorBlocks( peer: Peer, - needed: openarray[FetchRecord]) = + needed: openarray[FetchRecord]) {.libp2pProtocol("rpc/ancestor_blocks", "1.0.0").} = var resp = newSeqOfCap[BeaconBlock](needed.len) let db = peer.networkState.db var neededRoots = initSet[Eth2Digest]() diff --git a/beacon_chain/version.nim b/beacon_chain/version.nim index 146afd6263..753cd9cff7 100644 --- a/beacon_chain/version.nim +++ b/beacon_chain/version.nim @@ -1,5 +1,6 @@ const - useRLPx* = not defined(withLibP2P) + network_type {.strdefine.} = "rlpx" + useRLPx* = network_type == "rlpx" const versionMajor* = 0 diff --git a/nim.cfg b/nim.cfg index 049d31d895..869167e4fe 100644 --- a/nim.cfg +++ b/nim.cfg @@ -1,6 +1,8 @@ --threads:on --opt:speed +--define:"network_type=rlpx" + @if windows: # increase stack size --passL:"-Wl,--stack,8388608" diff --git a/scripts/testnet0.env b/scripts/testnet0.env index ebc41138cd..a678acef56 100644 --- a/scripts/testnet0.env +++ b/scripts/testnet0.env @@ -1,4 +1,4 @@ -NETWORK_ID=1000001 +NETWORK_ID=10 SHARD_COUNT=16 SLOTS_PER_EPOCH=16 SECONDS_PER_SLOT=30 diff --git a/scripts/testnet1.env b/scripts/testnet1.env index 9d9f6154e9..f35d9d3e9f 100644 --- a/scripts/testnet1.env +++ b/scripts/testnet1.env @@ -1,4 +1,4 @@ -NETWORK_ID=2000000 +NETWORK_ID=20 SHARD_COUNT=16 SLOTS_PER_EPOCH=16 SECONDS_PER_SLOT=30 From ea4690d5676aa61697eccc529073a10881e087ae Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Wed, 29 May 2019 11:21:03 +0300 Subject: [PATCH 02/18] A new P2P backend implementing the libp2p networking according to the official spec --- beacon_chain/libp2p_backend.nim | 7 +- beacon_chain/libp2p_spec_backend.nim | 664 +++++++++++++++++++++++++++ beacon_chain/ssz.nim | 4 + 3 files changed, 672 insertions(+), 3 deletions(-) create mode 100644 beacon_chain/libp2p_spec_backend.nim diff --git a/beacon_chain/libp2p_backend.nim b/beacon_chain/libp2p_backend.nim index c4a908079c..faf0a4358d 100644 --- a/beacon_chain/libp2p_backend.nim +++ b/beacon_chain/libp2p_backend.nim @@ -41,7 +41,7 @@ type name*: string # Private fields: - thunk*: MessageHandler + thunk*: ThunkProc libp2pProtocol: string printer*: MessageContentPrinter nextMsgResolver*: NextMsgResolver @@ -53,7 +53,7 @@ type NetworkStateInitializer* = proc(network: EthereumNode): RootRef {.gcsafe.} HandshakeStep* = proc(peer: Peer, handshakeStream: P2PStream): Future[void] {.gcsafe.} DisconnectionHandler* = proc(peer: Peer): Future[void] {.gcsafe.} - MessageHandler* = proc(daemon: DaemonAPI, stream: P2PStream): Future[void] {.gcsafe.} + ThunkProc* = proc(daemon: DaemonAPI, stream: P2PStream): Future[void] {.gcsafe.} MessageContentPrinter* = proc(msg: pointer): string {.gcsafe.} NextMsgResolver* = proc(msgData: SszReader, future: FutureBase) {.gcsafe.} @@ -297,7 +297,7 @@ proc setEventHandlers(p: ProtocolInfo, proc registerMsg(protocol: ProtocolInfo, name: string, - thunk: MessageHandler, + thunk: ThunkProc, libp2pProtocol: string, printer: MessageContentPrinter) = protocol.messages.add MessageInfo(name: name, @@ -373,6 +373,7 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = result.NetworkType = Eth2Node result.registerProtocol = bindSym "registerProtocol" result.setEventHandlers = bindSym "setEventHandlers" + result.SerializationFormat = Format result.afterProtocolInit = proc (p: P2PProtocol) = p.onPeerConnected.params.add newIdentDefs(ident"handshakeStream", P2PStream) diff --git a/beacon_chain/libp2p_spec_backend.nim b/beacon_chain/libp2p_spec_backend.nim new file mode 100644 index 0000000000..db406678ff --- /dev/null +++ b/beacon_chain/libp2p_spec_backend.nim @@ -0,0 +1,664 @@ +import + tables, deques, options, algorithm, std_shims/macros_shim, + chronos, chronicles, serialization, faststreams/input_stream, + eth/p2p/p2p_protocol_dsl, libp2p/daemon/daemonapi, + ssz + +const + # Compression nibble + NoCompression* = uint 0 + + # Encoding nibble + SszEncoding* = uint 1 + +type + Eth2Node* = ref object of RootObj + daemon*: DaemonAPI + peers*: Table[PeerID, Peer] + protocolStates*: seq[RootRef] + + EthereumNode = Eth2Node # needed for the definitions in p2p_backends_helpers + + ConnectionState* = enum + None, + Connecting, + Connected, + Disconnecting, + Disconnected + + DisconnectionReason* = enum + ClientShutdown = 1 + IrrelevantNetwork + FaultOrError + + CompressedMsgId = tuple + protocolIndex, msgId: int + + ResponseWithId*[MsgType] = object + peer*: Peer + id*: int + + Response*[MsgType] = distinct Peer + + # ----------------------------------------- + + ResponseCode* = enum + NoError + ParseError = 10 + InvalidRequest = 20 + MethodNotFound = 30 + ServerError = 40 + + OutstandingRequest* = object + id*: int + future*: FutureBase + timeoutAt*: Moment + + ProtocolConnection* = object + stream*: P2PStream + protocolInfo*: ProtocolInfo + + Peer* = ref object + network*: Eth2Node + id*: PeerID + lastSentMsgId*: int + rpcStream*: P2PStream + connectionState*: ConnectionState + protocolStates*: seq[RootRef] + maxInactivityAllowed: Duration + awaitedMessages: Table[CompressedMsgId, FutureBase] + outstandingRequests*: seq[Deque[OutstandingRequest]] + + MessageInfo* = object + id*: int + name*: string + + # Private fields: + thunk*: ThunkProc + libp2pProtocol: string + printer*: MessageContentPrinter + nextMsgResolver*: NextMsgResolver + requestResolver*: RequestResolver + + ProtocolInfoObj* = object + name*: string + version*: int + messages*: seq[MessageInfo] + index*: int # the position of the protocol in the + # ordered list of supported protocols + + # Private fields: + peerStateInitializer*: PeerStateInitializer + networkStateInitializer*: NetworkStateInitializer + handshake*: HandshakeStep + disconnectHandler*: DisconnectionHandler + dispatcher: Dispatcher + + ProtocolInfo* = ptr ProtocolInfoObj + + Dispatcher* = object + messages*: seq[MessageInfo] + + SpecOuterMsgHeader {.packed.} = object + compression {.bitsize: 4.}: uint + encoding {.bitsize: 4.}: uint + msgLen: uint64 + + SpecInnerMsgHeader {.packed.} = object + reqId: uint64 + methodId: uint16 + + PeerStateInitializer* = proc(peer: Peer): RootRef {.gcsafe.} + NetworkStateInitializer* = proc(network: Eth2Node): RootRef {.gcsafe.} + + HandshakeStep* = proc(peer: Peer, handshakeStream: P2PStream): Future[void] {.gcsafe.} + DisconnectionHandler* = proc(peer: Peer): Future[void] {.gcsafe.} + + ThunkProc* = proc(peer: Peer, + stream: P2PStream, + reqId: uint64, + msgData: ByteStreamVar): Future[void] {.gcsafe.} + + MessageContentPrinter* = proc(msg: pointer): string {.gcsafe.} + NextMsgResolver* = proc(msgData: SszReader, future: FutureBase) {.gcsafe.} + RequestResolver* = proc(msg: pointer, future: FutureBase) {.gcsafe.} + + Bytes = seq[byte] + + InvalidMsgIdError = object of InvalidMsgError + +var + gProtocols: seq[ProtocolInfo] + +include eth/p2p/p2p_backends_helpers +include eth/p2p/p2p_tracing + +proc `$`*(peer: Peer): string = $peer.id + +proc readFixedSizeStruct(stream: P2PStream, T: type): Future[T] {.async.} = + await stream.transp.readExactly(addr result, sizeof result) + +type + PeerLoopExitReason = enum + Success + UnsupportedCompression + UnsupportedEncoding + ProtocolViolation + InactivePeer + InternalError + +proc accepts(d: Dispatcher, methodId: uint16): bool = + methodId.int < d.messages.len + +proc invokeThunk(peer: Peer, + protocol: ProtocolInfo, + stream: P2PStream, + methodId: int, + reqId: uint64, + msgContents: ByteStreamVar): Future[void] = + template raiseInvalidMsgId = + raise newException(InvalidMsgIdError, + "ETH2 message with an invalid id " & $methodId) + + if methodId >= protocol.dispatcher.messages.len: raiseInvalidMsgId() + var thunk = protocol.dispatcher.messages[methodId].thunk + if thunk == nil: raiseInvalidMsgId() + + return thunk(peer, stream, reqId, msgContents) + +proc disconnect*(peer: Peer, reason: DisconnectionReason, notifyOtherPeer = false) {.async.} = + # TODO: How should we notify the other peer? + if peer.connectionState notin {Disconnecting, Disconnected}: + peer.connectionState = Disconnecting + await peer.network.daemon.disconnect(peer.id) + peer.connectionState = Disconnected + peer.network.peers.del(peer.id) + +proc recvAndDispatchMsg*(peer: Peer, protocol: ProtocolInfo, stream: P2PStream): + Future[PeerLoopExitReason] {.async.} = + template fail(reason) = + return reason + + var outerHeader = await stream.readFixedSizeStruct(SpecOuterMsgHeader) + + if outerHeader.compression != NoCompression: + fail UnsupportedCompression + + if outerHeader.encoding != SszEncoding: + fail UnsupportedEncoding + + if outerHeader.msgLen <= SpecInnerMsgHeader.sizeof.uint64: + fail ProtocolViolation + + var innerHeader = await stream.readFixedSizeStruct(SpecInnerMsgHeader) + + var msgContent = newSeq[byte](outerHeader.msgLen - SpecInnerMsgHeader.sizeof.uint64) + await stream.transp.readExactly(addr msgContent[0], msgContent.len) + + var msgContentStream = memoryStream(msgContent) + + if protocol.dispatcher.accepts(innerHeader.methodId): + try: + await invokeThunk(peer, protocol, stream, + innerHeader.methodId.int, + innerHeader.reqId, + msgContentStream) + except SerializationError: + fail ProtocolViolation + except CatchableError: + warn "" + +proc sendMsg*(peer: Peer, data: Bytes) {.gcsafe, async.} = + try: + var unsentBytes = data.len + while true: + unsentBytes -= await peer.rpcStream.transp.write(data) + if unsentBytes <= 0: return + except: + await peer.disconnect(FaultOrError) + # this is usually a "(32) Broken pipe": + # FIXME: this exception should be caught somewhere in addMsgHandler() and + # sending should be retried a few times + raise + +proc nextMsg*(peer: Peer, MsgType: type): Future[MsgType] = + ## This procs awaits a specific RLPx message. + ## Any messages received while waiting will be dispatched to their + ## respective handlers. The designated message handler will also run + ## to completion before the future returned by `nextMsg` is resolved. + let wantedId = peer.perPeerMsgId(MsgType) + let f = peer.awaitedMessages[wantedId] + if not f.isNil: + return Future[MsgType](f) + + initFuture result + peer.awaitedMessages[wantedId] = result + +proc dispatchMessages*(peer: Peer, protocol: ProtocolInfo, stream: P2PStream): + Future[PeerLoopExitReason] {.async.} = + while true: + let dispatchedMsgFut = recvAndDispatchMsg(peer, protocol, stream) + yield dispatchedMsgFut or sleepAsync(peer.maxInactivityAllowed) + if not dispatchedMsgFut.finished: + return InactivePeer + elif dispatchedMsgFut.failed: + error "Error in peer loop" + return InternalError + else: + let status = dispatchedMsgFut.read + if status == Success: continue + return status + +proc nextMsgResolver[MsgType](msgData: ByteStreamVar, future: FutureBase) {.gcsafe.} = + var reader = msgData + Future[MsgType](future).complete reader.readRecordType(MsgType, MsgType.rlpFieldsCount > 1) + +proc registerRequest(peer: Peer, + protocol: ProtocolInfo, + timeout: Duration, + responseFuture: FutureBase, + responseMsgId: int): int = + inc peer.lastSentMsgId + result = peer.lastSentMsgId + + let timeoutAt = Moment.fromNow(timeout) + let req = OutstandingRequest(id: result, + future: responseFuture, + timeoutAt: timeoutAt) + peer.outstandingRequests[responseMsgId].addLast req + + let requestResolver = protocol.dispatcher.messages[responseMsgId].requestResolver + proc timeoutExpired(udata: pointer) = requestResolver(nil, responseFuture) + + addTimer(timeoutAt, timeoutExpired, nil) + +proc resolveResponseFuture(peer: Peer, protocol: ProtocolInfo, msgId: int, msg: pointer, reqId: int) = + when false: + logScope: + msg = peer.dispatcher.messages[msgId].name + msgContents = peer.dispatcher.messages[msgId].printer(msg) + receivedReqId = reqId + remotePeer = peer.remote + + template resolve(future) = + (protocol.dispatcher.messages[msgId].requestResolver)(msg, future) + + template outstandingReqs: auto = + peer.outstandingRequests[msgId] + + # TODO: This is not completely sound because we are still using a global + # `reqId` sequence (the problem is that we might get a response ID that + # matches a request ID for a different type of request). To make the code + # correct, we can use a separate sequence per response type, but we have + # to first verify that the other Ethereum clients are supporting this + # correctly (because then, we'll be reusing the same reqIds for different + # types of requests). Alternatively, we can assign a separate interval in + # the `reqId` space for each type of response. + if reqId > peer.lastSentMsgId: + warn "RLPx response without a matching request" + return + + var idx = 0 + while idx < outstandingReqs.len: + template req: auto = outstandingReqs()[idx] + + if req.future.finished: + doAssert req.timeoutAt <= Moment.now() + # Here we'll remove the expired request by swapping + # it with the last one in the deque (if necessary): + if idx != outstandingReqs.len - 1: + req = outstandingReqs.popLast + continue + else: + outstandingReqs.shrink(fromLast = 1) + # This was the last item, so we don't have any + # more work to do: + return + + if req.id == reqId: + resolve req.future + # Here we'll remove the found request by swapping + # it with the last one in the deque (if necessary): + if idx != outstandingReqs.len - 1: + req = outstandingReqs.popLast + else: + outstandingReqs.shrink(fromLast = 1) + return + + inc idx + + debug "late or duplicate reply for a RLPx request" + +proc initProtocol(name: string, version: int, + peerInit: PeerStateInitializer, + networkInit: NetworkStateInitializer): ProtocolInfoObj = + result.name = name + result.version = version + result.messages = @[] + result.peerStateInitializer = peerInit + result.networkStateInitializer = networkInit + +proc setEventHandlers(p: ProtocolInfo, + handshake: HandshakeStep, + disconnectHandler: DisconnectionHandler) = + p.handshake = handshake + p.disconnectHandler = disconnectHandler + +proc registerProtocol(protocol: ProtocolInfo) = + # TODO: This can be done at compile-time in the future + let pos = lowerBound(gProtocols, protocol) + gProtocols.insert(protocol, pos) + for i in 0 ..< gProtocols.len: + gProtocols[i].index = i + +proc registerMsg(protocol: ProtocolInfo, + id: int, name: string, + thunk: ThunkProc, + printer: MessageContentPrinter, + requestResolver: RequestResolver, + nextMsgResolver: NextMsgResolver) = + if protocol.messages.len <= id: + protocol.messages.setLen(id + 1) + protocol.messages[id] = MessageInfo(id: id, + name: name, + thunk: thunk, + printer: printer, + requestResolver: requestResolver, + nextMsgResolver: nextMsgResolver) + +template applyDecorator(p: NimNode, decorator: NimNode) = + if decorator.kind != nnkNilLit: p.addPragma decorator + +proc implementSendProcBody(msg: Message): NimNode = + proc preludeGenerator(stream: NimNode): NimNode = + result = newStmtList() + if mgs.kind == msgRequest: + var reqId = ident "reqId" + let reqToResponseOffset = responseMsgId - msgId + let responseMsgId = quote do: `perPeerMsgIdVar` + `reqToResponseOffset` + + # Each request is registered so we can resolve it when the response + # arrives. There are two types of protocols: LES-like protocols use + # explicit `reqId` sent over the wire, while the ETH wire protocol + # assumes there is one outstanding request at a time (if there are + # multiple requests we'll resolve them in FIFO order). + let registerRequestCall = newCall(registerRequest, msgRecipient, + msg.timeoutParam[0], + resultIdent, + responseMsgId) + + result.add quote do: + let `reqId` = `registerRequestCall` + #`stream`.write(uint64(`reqId`)) + #`stream`.write(uint16(`methodId`)) + + proc sendCallGenerator(peer, bytes: NimNode): NimNode = + let + linkSendFailureToReqFuture = bindSym "linkSendFailureToReqFuture" + sendMsg = bindSym "sendMsg" + resultIdent = ident "result" + sendCall = newCall(sendMsg, peer, bytes) + + if msg.kind == msgRequest: + # In RLPx requests, the returned future was allocated here and passed + # to `registerRequest`. It's already assigned to the result variable + # of the proc, so we just wait for the sending operation to complete + # and we return in a normal way. (the waiting is done, so we can catch + # any possible errors). + quote: `linkSendFailureToReqFuture`(`sendCall`, `resultIdent`) + else: + # In normal RLPx messages, we are returning the future returned by the + # `sendMsg` call. + quote: return `sendCall` + + msg.createSendProcBody(preludeGenerator, sendCallGenerator) + +proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = + let + resultIdent = ident "result" + Option = bindSym "Option" + + # XXX: Binding the int type causes instantiation failure for some reason + # Int = bindSym "int" + Int = ident "int" + Peer = bindSym "Peer" + EthereumNode = bindSym "EthereumNode" + Format = bindSym "SSZ" + Response = bindSym "Response" + ResponseWithId = bindSym "ResponseWithId" + perProtocolMsgId = ident"perProtocolMsgId" + + mount = bindSym "mount" + + messagePrinter = bindSym "messagePrinter" + nextMsgResolver = bindSym "nextMsgResolver" + registerRequest = bindSym "registerRequest" + requestResolver = bindSym "requestResolver" + resolveResponseFuture = bindSym "resolveResponseFuture" + nextMsg = bindSym "nextMsg" + initProtocol = bindSym "initProtocol" + registerMsg = bindSym "registerMsg" + + peer = ident "peer" + reqId = ident "reqId" + stream = ident "stream" + protocol = ident "protocol" + msgContents = ident "msgContents" + receivedMsg = ident "receivedMsg" + + ProtocolInfo = bindSym "ProtocolInfo" + P2PStream = bindSym "P2PStream" + ByteStreamVar = bindSym "ByteStreamVar" + + new result + + result.registerProtocol = bindSym "registerProtocol" + result.setEventHandlers = bindSym "setEventHandlers" + result.PeerType = Peer + result.NetworkType = EthereumNode + result.SerializationFormat = Format + + result.implementMsg = proc (p: P2PProtocol, msg: Message, resp: Message = nil) = + var + msgId = msg.id + msgIdLit = newLit(msgId) + msgRecName = msg.recIdent + msgKind = msg.kind + n = msg.procDef + responseMsgId = if resp != nil: resp.id else: -1 + responseRecord = if resp != nil: resp.recIdent else: nil + msgIdent = n.name + msgName = $msgIdent + hasReqIds = msgKind in {msgRequest, msgResponse} + userPragmas = n.pragma + + # variables used in the sending procs + msgRecipient = ident"msgRecipient" + sendTo = ident"sendTo" + rlpWriter = ident"writer" + paramsToWrite = newSeq[NimNode](0) + perPeerMsgIdVar = ident"perPeerMsgId" + + # nodes to store the user-supplied message handling proc if present + userHandlerCall: NimNode = nil + awaitUserHandler = newStmtList() + + case msgKind + of msgRequest: + discard + + of msgResponse: + if hasReqIds: + paramsToWrite.add newDotExpr(sendTo, ident"id") + + of msgHandshake, msgNotification: discard + + if msg.userHandler != nil: + var extraDefs: NimNode + if msgKind == msgRequest: + let peerParam = msg.userHandler.params[1][0] + let response = ident"response" + if hasReqIds: + extraDefs = quote do: + let `response` = `ResponseWithId`[`responseRecord`](peer: `peerParam`, id: `reqId`) + else: + extraDefs = quote do: + let `response` = `Response`[`responseRecord`](`peerParam`) + + msg.userHandler.addPreludeDefs extraDefs + + # This is the call to the user supplied handled. Here we add only the + # initial peer param, while the rest of the params will be added later. + userHandlerCall = newCall(msg.userHandler.name, peer) + + if hasReqIds: + msg.userHandler.params.insert(2, newIdentDefs(reqId, ident"int")) + userHandlerCall.add reqId + + # When there is a user handler, it must be awaited in the thunk proc. + # Above, by default `awaitUserHandler` is set to a no-op statement list. + awaitUserHandler = newCall("await", userHandlerCall) + + p.outRecvProcs.add(msg.userHandler) + + for param, paramType in n.typedParams(skip = 1): + # This is a fragment of the sending proc that + # serializes each of the passed parameters: + paramsToWrite.add param + + # If there is user message handler, we'll place a call to it by + # unpacking the fields of the received message: + if userHandlerCall != nil: + userHandlerCall.add newDotExpr(receivedMsg, param) + + let traceMsg = when tracingEnabled: + newCall(bindSym"logReceivedMsg", peer, receivedMsg) + else: + newStmtList() + + # variables used in the receiving procs + let callResolvedResponseFuture = if msgKind == msgResponse: + newCall(resolveResponseFuture, peer, msgIdLit, newCall("addr", receivedMsg), reqId) + else: + newStmtList() + + let thunkName = ident(msgName & "_thunk") + var thunkProc = quote do: + proc `thunkName`(`peer`: `Peer`, + `stream`: `P2PStream`, + `reqId`: uint64, + `msgContents`: `ByteStreamVar`) {.async, gcsafe.} = + var `receivedMsg` = `mount`(`SSZ`, `msgContents`, `msgRecName`) + `traceMsg` + `awaitUserHandler` + `callResolvedResponseFuture` + + for p in userPragmas: thunkProc.addPragma p + + case msgKind + of msgRequest: thunkProc.applyDecorator p.incomingRequestThunkDecorator + of msgResponse: thunkProc.applyDecorator p.incomingResponseThunkDecorator + else: discard + + p.outRecvProcs.add thunkProc + + var msgSendProc = n + let msgSendProcName = n.name + p.outSendProcs.add msgSendProc + + # TODO: check that the first param has the correct type + msgSendProc.params[1][0] = sendTo + msgSendProc.addPragma ident"gcsafe" + + case msgKind + of msgRequest: + # Add a timeout parameter for all request procs + msgSendProc.params.add msg.timeoutParam + of msgResponse: + # A response proc must be called with a response object that originates + # from a certain request. Here we change the Peer parameter at position + # 1 to the correct strongly-typed ResponseType. The incoming procs still + # gets the normal Peer paramter. + # let rsp = bindSym "Response" + # let rspId = bindSym "ResponseWithId" + let + ResponseType = newTree(nnkBracketExpr, ResponseWithId, msgRecName) + + msgSendProc.params[1][1] = ResponseType + + p.outSendProcs.add quote do: + template send*(r: `ResponseType`, args: varargs[untyped]): auto = + `msgSendProcName`(r, args) + else: discard + + # We change the return type of the sending proc to a Future. + # If this is a request proc, the future will return the response record. + let rt = if msgKind != msgRequest: ident"void" + else: newTree(nnkBracketExpr, Option, responseRecord) + msgSendProc.params[0] = newTree(nnkBracketExpr, ident("Future"), rt) + + let msgBytes = ident"msgBytes" + + let finalizeRequest = quote do: + let `msgBytes` = `finish`(`rlpWriter`) + + if msgKind == msgHandshake: + var + rawSendProc = genSym(nskProc, msgName & "RawSend") + handshakeExchanger = newProc(name = msg.identWithExportMarker, + procType = nnkTemplateDef) + + handshakeExchanger.params = msgSendProc.params.copyNimTree + handshakeExchanger.params.add msg.timeoutParam + handshakeExchanger.params[0] = newTree(nnkBracketExpr, ident("Future"), msgRecName) + + var + forwardCall = newCall(rawSendProc).appendAllParams(handshakeExchanger) + peerValue = forwardCall[1] + timeoutValue = msg.timeoutParam[0] + handshakeImpl = ident"handshakeImpl" + + forwardCall[1] = peer + forwardCall.del(forwardCall.len - 1) + + handshakeExchanger.body = quote do: + let `peer` = `peerValue` + let sendingFuture = `forwardCall` + `handshakeImpl`(`peer`, + sendingFuture, + `nextMsg`(`peer`, `msgRecName`), + `timeoutValue`) + + msgSendProc.name = rawSendProc + p.outSendProcs.add handshakeExchanger + else: + # Make the send proc public + msgSendProc.name = msg.identWithExportMarker + + let initWriter = quote do: + var `rlpWriter` = `initRlpWriter`() + const `perProtocolMsgId` = `msgId` + let `perPeerMsgIdVar` = `msgIdLit` + `append`(`rlpWriter`, `perPeerMsgIdVar`) + + msgSendProc.body = implementSendProcBody(msg) + + if msgKind == msgRequest: + msgSendProc.applyDecorator p.outgoingRequestDecorator + + p.outProcRegistrations.add( + newCall(registerMsg, + p.protocolInfoVar, + newIntLitNode(msgId), + newStrLitNode($n.name), + thunkName, + newTree(nnkBracketExpr, messagePrinter, msgRecName), + newTree(nnkBracketExpr, requestResolver, msgRecName), + newTree(nnkBracketExpr, nextMsgResolver, msgRecName))) + + result.implementProtocolInit = proc (p: P2PProtocol): NimNode = + return newCall(initProtocol, + newLit(p.shortName), + newLit(p.version), + p.peerInit, p.netInit) + diff --git a/beacon_chain/ssz.nim b/beacon_chain/ssz.nim index 2d49330ee7..69c686f6d2 100644 --- a/beacon_chain/ssz.nim +++ b/beacon_chain/ssz.nim @@ -40,6 +40,10 @@ serializationFormat SSZ, proc init*(T: type SszReader, stream: ByteStreamVar): T = result.stream = stream +proc mount*(F: type SSZ, stream: ByteStreamVar, T: type): T = + mixin readValue + init(SszReader, stream).readValue(T) + func toSSZType(x: Slot|Epoch): auto = x.uint64 func toSSZType(x: auto): auto = x From c060c0fc5dd1348866e95cfd3c381ef6625658b9 Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Wed, 29 May 2019 18:55:25 +0300 Subject: [PATCH 03/18] Simplified the 'spec' back-end by movign more logic into the shared DSL module --- beacon_chain/libp2p_backend.nim | 13 +- beacon_chain/libp2p_spec_backend.nim | 301 +++++++++++---------------- 2 files changed, 129 insertions(+), 185 deletions(-) diff --git a/beacon_chain/libp2p_backend.nim b/beacon_chain/libp2p_backend.nim index faf0a4358d..96bd3552a0 100644 --- a/beacon_chain/libp2p_backend.nim +++ b/beacon_chain/libp2p_backend.nim @@ -275,12 +275,6 @@ template getRecipient(stream: P2PStream): P2PStream = template getRecipient(response: Response): Peer = UntypedResponse(response).peer -proc messagePrinter[MsgType](msg: pointer): string {.gcsafe.} = - result = "" - # TODO: uncommenting the line below increases the compile-time - # tremendously (for reasons not yet known) - # result = $(cast[ptr MsgType](msg)[]) - proc initProtocol(name: string, peerInit: PeerStateInitializer, networkInit: NetworkStateInitializer): ProtocolInfoObj = @@ -374,6 +368,7 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = result.registerProtocol = bindSym "registerProtocol" result.setEventHandlers = bindSym "setEventHandlers" result.SerializationFormat = Format + result.ResponseType = Response result.afterProtocolInit = proc (p: P2PProtocol) = p.onPeerConnected.params.add newIdentDefs(ident"handshakeStream", P2PStream) @@ -501,13 +496,13 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = rawSendProc = msgName & "RawSend" handshakeTypeName = $msgRecName handshakeExchanger = msg.createSendProc(nnkMacroDef) - paramsArray = newTree(nnkBracket).appendAllParams(handshakeExchanger) + paramsArray = newTree(nnkBracket).appendAllParams(handshakeExchanger.def) bindSym = ident "bindSym" getAst = ident "getAst" handshakeImpl = ident "handshakeImpl" # TODO: macros.body triggers an assertion error when the proc type is nnkMacroDef - handshakeExchanger[6] = quote do: + handshakeExchanger.def[6] = quote do: let stream = ident"handshakeStream" rawSendProc = `bindSymOp` `rawSendProc` @@ -521,7 +516,7 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = return `getAst`(`handshakeImpl`(`msgRecName`, peer, stream, lazySendCall, timeout)) - p.outSendProcs.add handshakeExchanger + p.outSendProcs.add handshakeExchanger.def msgSendProc.params[1][1] = P2PStream msgSendProc.name = ident rawSendProc diff --git a/beacon_chain/libp2p_spec_backend.nim b/beacon_chain/libp2p_spec_backend.nim index db406678ff..2cb99ce4b7 100644 --- a/beacon_chain/libp2p_spec_backend.nim +++ b/beacon_chain/libp2p_spec_backend.nim @@ -7,7 +7,7 @@ import const # Compression nibble NoCompression* = uint 0 - + # Encoding nibble SszEncoding* = uint 1 @@ -79,7 +79,7 @@ type printer*: MessageContentPrinter nextMsgResolver*: NextMsgResolver requestResolver*: RequestResolver - + ProtocolInfoObj* = object name*: string version*: int @@ -110,10 +110,10 @@ type PeerStateInitializer* = proc(peer: Peer): RootRef {.gcsafe.} NetworkStateInitializer* = proc(network: Eth2Node): RootRef {.gcsafe.} - + HandshakeStep* = proc(peer: Peer, handshakeStream: P2PStream): Future[void] {.gcsafe.} DisconnectionHandler* = proc(peer: Peer): Future[void] {.gcsafe.} - + ThunkProc* = proc(peer: Peer, stream: P2PStream, reqId: uint64, @@ -127,17 +127,25 @@ type InvalidMsgIdError = object of InvalidMsgError + PeerDisconnected* = object of P2PBackendError + reason*: DisconnectionReason + var gProtocols: seq[ProtocolInfo] -include eth/p2p/p2p_backends_helpers -include eth/p2p/p2p_tracing +const + HandshakeTimeout = FaultOrError + # TODO: this doesn't seem right. + # We should lobby for more disconnection reasons. proc `$`*(peer: Peer): string = $peer.id -proc readFixedSizeStruct(stream: P2PStream, T: type): Future[T] {.async.} = +proc readPackedObject(stream: P2PStream, T: type): Future[T] {.async.} = await stream.transp.readExactly(addr result, sizeof result) +proc appendPackedObject*(stream: ByteStreamVar, value: auto) = + stream.append makeOpenArray(unsafeAddr value, sizeof value) + type PeerLoopExitReason = enum Success @@ -147,6 +155,29 @@ type InactivePeer InternalError +proc disconnect*(peer: Peer, reason: DisconnectionReason, notifyOtherPeer = false) {.async.} = + # TODO: How should we notify the other peer? + if peer.connectionState notin {Disconnecting, Disconnected}: + peer.connectionState = Disconnecting + await peer.network.daemon.disconnect(peer.id) + peer.connectionState = Disconnected + peer.network.peers.del(peer.id) + +template raisePeerDisconnected(msg: string, r: DisconnectionReason) = + var e = newException(PeerDisconnected, msg) + e.reason = r + raise e + +proc disconnectAndRaise(peer: Peer, + reason: DisconnectionReason, + msg: string) {.async.} = + let r = reason + await peer.disconnect(r) + raisePeerDisconnected(msg, r) + +include eth/p2p/p2p_backends_helpers +include eth/p2p/p2p_tracing + proc accepts(d: Dispatcher, methodId: uint16): bool = methodId.int < d.messages.len @@ -166,31 +197,23 @@ proc invokeThunk(peer: Peer, return thunk(peer, stream, reqId, msgContents) -proc disconnect*(peer: Peer, reason: DisconnectionReason, notifyOtherPeer = false) {.async.} = - # TODO: How should we notify the other peer? - if peer.connectionState notin {Disconnecting, Disconnected}: - peer.connectionState = Disconnecting - await peer.network.daemon.disconnect(peer.id) - peer.connectionState = Disconnected - peer.network.peers.del(peer.id) - proc recvAndDispatchMsg*(peer: Peer, protocol: ProtocolInfo, stream: P2PStream): Future[PeerLoopExitReason] {.async.} = template fail(reason) = return reason - var outerHeader = await stream.readFixedSizeStruct(SpecOuterMsgHeader) + var outerHeader = await stream.readPackedObject(SpecOuterMsgHeader) if outerHeader.compression != NoCompression: fail UnsupportedCompression - + if outerHeader.encoding != SszEncoding: fail UnsupportedEncoding if outerHeader.msgLen <= SpecInnerMsgHeader.sizeof.uint64: fail ProtocolViolation - var innerHeader = await stream.readFixedSizeStruct(SpecInnerMsgHeader) + var innerHeader = await stream.readPackedObject(SpecInnerMsgHeader) var msgContent = newSeq[byte](outerHeader.msgLen - SpecInnerMsgHeader.sizeof.uint64) await stream.transp.readExactly(addr msgContent[0], msgContent.len) @@ -257,7 +280,7 @@ proc registerRequest(peer: Peer, protocol: ProtocolInfo, timeout: Duration, responseFuture: FutureBase, - responseMsgId: int): int = + responseMethodId: uint16): int = inc peer.lastSentMsgId result = peer.lastSentMsgId @@ -265,11 +288,11 @@ proc registerRequest(peer: Peer, let req = OutstandingRequest(id: result, future: responseFuture, timeoutAt: timeoutAt) - peer.outstandingRequests[responseMsgId].addLast req + peer.outstandingRequests[responseMethodId.int].addLast req - let requestResolver = protocol.dispatcher.messages[responseMsgId].requestResolver + let requestResolver = protocol.dispatcher.messages[responseMethodId.int].requestResolver proc timeoutExpired(udata: pointer) = requestResolver(nil, responseFuture) - + addTimer(timeoutAt, timeoutExpired, nil) proc resolveResponseFuture(peer: Peer, protocol: ProtocolInfo, msgId: int, msg: pointer, reqId: int) = @@ -369,34 +392,48 @@ proc registerMsg(protocol: ProtocolInfo, template applyDecorator(p: NimNode, decorator: NimNode) = if decorator.kind != nnkNilLit: p.addPragma decorator -proc implementSendProcBody(msg: Message): NimNode = +proc prepareRequest(peer: Peer, + protocol: ProtocolInfo, + requestMethodId, responseMethodId: uint16, + stream: ByteStreamVar, + timeout: Duration, + responseFuture: FutureBase): DelayedWriteCursor = + + let reqId = registerRequest(peer, protocol, timeout, + responseFuture, responseMethodId) + + result = stream.delayFixedSizeWrite sizeof(SpecOuterMsgHeader) + + stream.appendPackedObject SpecInnerMsgHeader( + reqId: reqId, + methodId: requestMethodId) + +proc implementSendProcBody(sendProc: SendProc) = + let + msg = sendProc.msg + resultIdent = ident "result" + delayedWriteCursor = ident "delayedWriteCursor" + proc preludeGenerator(stream: NimNode): NimNode = result = newStmtList() - if mgs.kind == msgRequest: - var reqId = ident "reqId" - let reqToResponseOffset = responseMsgId - msgId - let responseMsgId = quote do: `perPeerMsgIdVar` + `reqToResponseOffset` - - # Each request is registered so we can resolve it when the response - # arrives. There are two types of protocols: LES-like protocols use - # explicit `reqId` sent over the wire, while the ETH wire protocol - # assumes there is one outstanding request at a time (if there are - # multiple requests we'll resolve them in FIFO order). - let registerRequestCall = newCall(registerRequest, msgRecipient, - msg.timeoutParam[0], - resultIdent, - responseMsgId) + if msg.kind == msgRequest: + let + reqId = ident "reqId" + appendPackedObject = bindSym "appendPackedObject" + requestMethodId = newLit(msg.id) + responseMethodId = newLit(msg.response.id) + peer = sendProc.peerParam + protocol = sendProc.msg.protocol.protocolInfoVar + timeout = sendProc.timeoutParam result.add quote do: - let `reqId` = `registerRequestCall` - #`stream`.write(uint64(`reqId`)) - #`stream`.write(uint16(`methodId`)) + let `delayedWriteCursor` = `prepareRequest`( + `peer`, `protocol`, `requestMethodId`, `responseMethodId`, `stream`, `timeout`, `resultIdent`) proc sendCallGenerator(peer, bytes: NimNode): NimNode = let linkSendFailureToReqFuture = bindSym "linkSendFailureToReqFuture" sendMsg = bindSym "sendMsg" - resultIdent = ident "result" sendCall = newCall(sendMsg, peer, bytes) if msg.kind == msgRequest: @@ -411,13 +448,13 @@ proc implementSendProcBody(msg: Message): NimNode = # `sendMsg` call. quote: return `sendCall` - msg.createSendProcBody(preludeGenerator, sendCallGenerator) + sendProc.implementBody(preludeGenerator, sendCallGenerator) proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = let resultIdent = ident "result" Option = bindSym "Option" - + # XXX: Binding the int type causes instantiation failure for some reason # Int = bindSym "int" Int = ident "int" @@ -432,17 +469,17 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = messagePrinter = bindSym "messagePrinter" nextMsgResolver = bindSym "nextMsgResolver" - registerRequest = bindSym "registerRequest" requestResolver = bindSym "requestResolver" resolveResponseFuture = bindSym "resolveResponseFuture" nextMsg = bindSym "nextMsg" initProtocol = bindSym "initProtocol" registerMsg = bindSym "registerMsg" - + peer = ident "peer" reqId = ident "reqId" stream = ident "stream" protocol = ident "protocol" + response = ident "response" msgContents = ident "msgContents" receivedMsg = ident "receivedMsg" @@ -457,16 +494,16 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = result.PeerType = Peer result.NetworkType = EthereumNode result.SerializationFormat = Format + result.ResponseType = ResponseWithId - result.implementMsg = proc (p: P2PProtocol, msg: Message, resp: Message = nil) = + result.implementMsg = proc (protocol: P2PProtocol, msg: Message) = var - msgId = msg.id - msgIdLit = newLit(msgId) + msgIdLit = newLit(msg.id) msgRecName = msg.recIdent msgKind = msg.kind n = msg.procDef - responseMsgId = if resp != nil: resp.id else: -1 - responseRecord = if resp != nil: resp.recIdent else: nil + responseMethodId = if msg.response != nil: msg.response.id else: -1 + responseRecord = if msg.response != nil: msg.response.recIdent else: nil msgIdent = n.name msgName = $msgIdent hasReqIds = msgKind in {msgRequest, msgResponse} @@ -474,75 +511,43 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = # variables used in the sending procs msgRecipient = ident"msgRecipient" - sendTo = ident"sendTo" rlpWriter = ident"writer" paramsToWrite = newSeq[NimNode](0) perPeerMsgIdVar = ident"perPeerMsgId" - # nodes to store the user-supplied message handling proc if present - userHandlerCall: NimNode = nil - awaitUserHandler = newStmtList() - - case msgKind - of msgRequest: - discard - - of msgResponse: - if hasReqIds: - paramsToWrite.add newDotExpr(sendTo, ident"id") - - of msgHandshake, msgNotification: discard - + ## + ## Augment user handler + ## if msg.userHandler != nil: - var extraDefs: NimNode if msgKind == msgRequest: + msg.userHandler.params.insert(2, newIdentDefs(reqId, Int)) + let peerParam = msg.userHandler.params[1][0] - let response = ident"response" - if hasReqIds: - extraDefs = quote do: - let `response` = `ResponseWithId`[`responseRecord`](peer: `peerParam`, id: `reqId`) - else: - extraDefs = quote do: - let `response` = `Response`[`responseRecord`](`peerParam`) + let extraDefs = quote do: + let `response` = `ResponseWithId`[`responseRecord`](peer: `peerParam`, id: `reqId`) msg.userHandler.addPreludeDefs extraDefs - # This is the call to the user supplied handled. Here we add only the - # initial peer param, while the rest of the params will be added later. - userHandlerCall = newCall(msg.userHandler.name, peer) - - if hasReqIds: - msg.userHandler.params.insert(2, newIdentDefs(reqId, ident"int")) - userHandlerCall.add reqId - - # When there is a user handler, it must be awaited in the thunk proc. - # Above, by default `awaitUserHandler` is set to a no-op statement list. - awaitUserHandler = newCall("await", userHandlerCall) - - p.outRecvProcs.add(msg.userHandler) - - for param, paramType in n.typedParams(skip = 1): - # This is a fragment of the sending proc that - # serializes each of the passed parameters: - paramsToWrite.add param - - # If there is user message handler, we'll place a call to it by - # unpacking the fields of the received message: - if userHandlerCall != nil: - userHandlerCall.add newDotExpr(receivedMsg, param) + protocol.outRecvProcs.add(msg.userHandler) + ## + ## Implemenmt Thunk + ## let traceMsg = when tracingEnabled: newCall(bindSym"logReceivedMsg", peer, receivedMsg) else: newStmtList() - + # variables used in the receiving procs let callResolvedResponseFuture = if msgKind == msgResponse: newCall(resolveResponseFuture, peer, msgIdLit, newCall("addr", receivedMsg), reqId) else: newStmtList() - let thunkName = ident(msgName & "_thunk") + let + awaitUserHandler = msg.genAwaitUserHandler(receivedMsg, peer, reqId) + thunkName = ident(msgName & "_thunk") + var thunkProc = quote do: proc `thunkName`(`peer`: `Peer`, `stream`: `P2PStream`, @@ -553,67 +558,23 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = `awaitUserHandler` `callResolvedResponseFuture` - for p in userPragmas: thunkProc.addPragma p - - case msgKind - of msgRequest: thunkProc.applyDecorator p.incomingRequestThunkDecorator - of msgResponse: thunkProc.applyDecorator p.incomingResponseThunkDecorator - else: discard - - p.outRecvProcs.add thunkProc - - var msgSendProc = n - let msgSendProcName = n.name - p.outSendProcs.add msgSendProc - - # TODO: check that the first param has the correct type - msgSendProc.params[1][0] = sendTo - msgSendProc.addPragma ident"gcsafe" - - case msgKind - of msgRequest: - # Add a timeout parameter for all request procs - msgSendProc.params.add msg.timeoutParam - of msgResponse: - # A response proc must be called with a response object that originates - # from a certain request. Here we change the Peer parameter at position - # 1 to the correct strongly-typed ResponseType. The incoming procs still - # gets the normal Peer paramter. - # let rsp = bindSym "Response" - # let rspId = bindSym "ResponseWithId" - let - ResponseType = newTree(nnkBracketExpr, ResponseWithId, msgRecName) + protocol.outRecvProcs.add thunkProc - msgSendProc.params[1][1] = ResponseType + ## + ## Implement Senders and Handshake + ## + var sendProc = createSendProc(msg, isRawSender = (msg.kind == msgHandshake)) + protocol.outSendProcs.add sendProc.allDefs - p.outSendProcs.add quote do: - template send*(r: `ResponseType`, args: varargs[untyped]): auto = - `msgSendProcName`(r, args) - else: discard - - # We change the return type of the sending proc to a Future. - # If this is a request proc, the future will return the response record. - let rt = if msgKind != msgRequest: ident"void" - else: newTree(nnkBracketExpr, Option, responseRecord) - msgSendProc.params[0] = newTree(nnkBracketExpr, ident("Future"), rt) - - let msgBytes = ident"msgBytes" - - let finalizeRequest = quote do: - let `msgBytes` = `finish`(`rlpWriter`) - - if msgKind == msgHandshake: + if msg.kind == msgHandshake: var rawSendProc = genSym(nskProc, msgName & "RawSend") - handshakeExchanger = newProc(name = msg.identWithExportMarker, - procType = nnkTemplateDef) + handshakeExchanger = createSendProc(msg, procType = nnkTemplateDef) - handshakeExchanger.params = msgSendProc.params.copyNimTree - handshakeExchanger.params.add msg.timeoutParam - handshakeExchanger.params[0] = newTree(nnkBracketExpr, ident("Future"), msgRecName) + handshakeExchanger.def.params[0] = newTree(nnkBracketExpr, ident("Future"), msgRecName) var - forwardCall = newCall(rawSendProc).appendAllParams(handshakeExchanger) + forwardCall = newCall(rawSendProc).appendAllParams(handshakeExchanger.def) peerValue = forwardCall[1] timeoutValue = msg.timeoutParam[0] handshakeImpl = ident"handshakeImpl" @@ -621,7 +582,7 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = forwardCall[1] = peer forwardCall.del(forwardCall.len - 1) - handshakeExchanger.body = quote do: + handshakeExchanger.def.body = quote do: let `peer` = `peerValue` let sendingFuture = `forwardCall` `handshakeImpl`(`peer`, @@ -629,36 +590,24 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = `nextMsg`(`peer`, `msgRecName`), `timeoutValue`) - msgSendProc.name = rawSendProc - p.outSendProcs.add handshakeExchanger + sendProc.def.name = rawSendProc + protocol.outSendProcs.add handshakeExchanger.def else: - # Make the send proc public - msgSendProc.name = msg.identWithExportMarker - - let initWriter = quote do: - var `rlpWriter` = `initRlpWriter`() - const `perProtocolMsgId` = `msgId` - let `perPeerMsgIdVar` = `msgIdLit` - `append`(`rlpWriter`, `perPeerMsgIdVar`) - - msgSendProc.body = implementSendProcBody(msg) - - if msgKind == msgRequest: - msgSendProc.applyDecorator p.outgoingRequestDecorator + implementSendProcBody sendProc - p.outProcRegistrations.add( + protocol.outProcRegistrations.add( newCall(registerMsg, - p.protocolInfoVar, - newIntLitNode(msgId), - newStrLitNode($n.name), + protocol.protocolInfoVar, + msgIdLit, + newLit(msgName), thunkName, newTree(nnkBracketExpr, messagePrinter, msgRecName), newTree(nnkBracketExpr, requestResolver, msgRecName), newTree(nnkBracketExpr, nextMsgResolver, msgRecName))) - result.implementProtocolInit = proc (p: P2PProtocol): NimNode = + result.implementProtocolInit = proc (protocol: P2PProtocol): NimNode = return newCall(initProtocol, - newLit(p.shortName), - newLit(p.version), - p.peerInit, p.netInit) + newLit(protocol.shortName), + newLit(protocol.version), + protocol.peerInit, protocol.netInit) From e177d177621a3512b5ce6c74911dd4a0a352aaa2 Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Thu, 30 May 2019 21:57:12 +0300 Subject: [PATCH 04/18] Further simplifications --- beacon_chain/libp2p_backend.nim | 39 +++++----- beacon_chain/libp2p_spec_backend.nim | 102 +++++++-------------------- 2 files changed, 44 insertions(+), 97 deletions(-) diff --git a/beacon_chain/libp2p_backend.nim b/beacon_chain/libp2p_backend.nim index 96bd3552a0..0ce4a48958 100644 --- a/beacon_chain/libp2p_backend.nim +++ b/beacon_chain/libp2p_backend.nim @@ -68,7 +68,7 @@ type peer*: Peer stream*: P2PStream - Response*[MsgType] = distinct UntypedResponse + Responder*[MsgType] = distinct UntypedResponse Bytes = seq[byte] @@ -82,6 +82,7 @@ type const defaultIncomingReqTimeout = 5000 defaultOutgoingReqTimeout = 10000 + HandshakeTimeout = BreachOfProtocol var gProtocols: seq[ProtocolInfo] @@ -272,7 +273,7 @@ template getRecipient(peer: Peer): Peer = template getRecipient(stream: P2PStream): P2PStream = stream -template getRecipient(response: Response): Peer = +template getRecipient(response: Responder): Peer = UntypedResponse(response).peer proc initProtocol(name: string, @@ -329,7 +330,7 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = Format = ident"SSZ" Option = bindSym "Option" UntypedResponse = bindSym "UntypedResponse" - Response = bindSym "Response" + Responder = bindSym "Responder" DaemonAPI = bindSym "DaemonAPI" P2PStream = ident "P2PStream" # XXX: Binding the int type causes instantiation failure for some reason @@ -368,20 +369,21 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = result.registerProtocol = bindSym "registerProtocol" result.setEventHandlers = bindSym "setEventHandlers" result.SerializationFormat = Format - result.ResponseType = Response + result.ResponderType = Responder result.afterProtocolInit = proc (p: P2PProtocol) = p.onPeerConnected.params.add newIdentDefs(ident"handshakeStream", P2PStream) - result.implementMsg = proc (p: P2PProtocol, msg: Message, resp: Message = nil) = + result.implementMsg = proc (msg: Message) = let n = msg.procDef + protocol = msg.protocol msgId = newLit(msg.id) msgIdent = n.name msgName = $msgIdent msgKind = msg.kind msgRecName = msg.recIdent - responseRecord = if resp != nil: resp.recIdent else: nil + ResponseRecord = if msg.response != nil: msg.response.recIdent else: nil userPragmas = n.pragma var @@ -415,14 +417,14 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = extraDefs = quote do: # Jump through some hoops to work aroung # https://github.com/nim-lang/Nim/issues/6248 - let `response` = `Response`[`responseRecord`]( + let `response` = `Responder`[`ResponseRecord`]( `UntypedResponse`(peer: `peer`, stream: `stream`)) # Resolve the Eth2Peer from the LibP2P data received in the thunk userHandlerCall.add peerIdent msg.userHandler.addPreludeDefs extraDefs - p.outRecvProcs.add msg.userHandler + protocol.outRecvProcs.add msg.userHandler elif msgName == "status": #awaitUserHandler = quote do: @@ -455,14 +457,11 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = `awaitUserHandler` `resolveNextMsgFutures`(`peerIdent`, get(`receivedMsg`)) - for p in userPragmas: - thunkProc.addPragma p - - p.outRecvProcs.add thunkProc + protocol.outRecvProcs.add thunkProc var msgSendProc = n let msgSendProcName = n.name - p.outSendProcs.add msgSendProc + protocol.outSendProcs.add msgSendProc # TODO: check that the first param has the correct type msgSendProc.params[1][0] = sendTo @@ -478,9 +477,9 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = # from a certain request. Here we change the Peer parameter at position # 1 to the correct strongly-typed ResponseType. The incoming procs still # gets the normal Peer paramter. - let ResponseType = newTree(nnkBracketExpr, Response, msgRecName) + let ResponseType = newTree(nnkBracketExpr, Responder, msgRecName) msgSendProc.params[1][1] = ResponseType - p.outSendProcs.add quote do: + protocol.outSendProcs.add quote do: template send*(r: `ResponseType`, args: varargs[untyped]): auto = `msgSendProcName`(r, args) else: discard @@ -488,7 +487,7 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = # We change the return type of the sending proc to a Future. # If this is a request proc, the future will return the response record. let rt = if msgKind != msgRequest: Void - else: newTree(nnkBracketExpr, Option, responseRecord) + else: newTree(nnkBracketExpr, Option, ResponseRecord) msgSendProc.params[0] = newTree(nnkBracketExpr, ident("Future"), rt) if msgKind == msgHandshake: @@ -516,7 +515,7 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = return `getAst`(`handshakeImpl`(`msgRecName`, peer, stream, lazySendCall, timeout)) - p.outSendProcs.add handshakeExchanger.def + protocol.outSendProcs.add handshakeExchanger.def msgSendProc.params[1][1] = P2PStream msgSendProc.name = ident rawSendProc @@ -553,7 +552,7 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = if msgKind == msgRequest: let timeout = msg.timeoutParam[0] quote: `makeEth2Request`(`msgRecipient`, `msgProto`, `msgBytes`, - `responseRecord`, `timeout`) + `ResponseRecord`, `timeout`) elif msgId.intVal == 0: quote: `sendBytes`(`sendTo`, `msgBytes`) else: @@ -568,9 +567,9 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = `finalizeRequest` return `sendCall` - p.outProcRegistrations.add( + protocol.outProcRegistrations.add( newCall(registerMsg, - p.protocolInfoVar, + protocol.protocolInfoVar, newLit(msgName), thunkName, msgProto, diff --git a/beacon_chain/libp2p_spec_backend.nim b/beacon_chain/libp2p_spec_backend.nim index 2cb99ce4b7..395a52ba46 100644 --- a/beacon_chain/libp2p_spec_backend.nim +++ b/beacon_chain/libp2p_spec_backend.nim @@ -34,7 +34,7 @@ type CompressedMsgId = tuple protocolIndex, msgId: int - ResponseWithId*[MsgType] = object + ResponderWithId*[MsgType] = object peer*: Peer id*: int @@ -411,15 +411,12 @@ proc prepareRequest(peer: Peer, proc implementSendProcBody(sendProc: SendProc) = let msg = sendProc.msg - resultIdent = ident "result" delayedWriteCursor = ident "delayedWriteCursor" proc preludeGenerator(stream: NimNode): NimNode = result = newStmtList() if msg.kind == msgRequest: let - reqId = ident "reqId" - appendPackedObject = bindSym "appendPackedObject" requestMethodId = newLit(msg.id) responseMethodId = newLit(msg.response.id) peer = sendProc.peerParam @@ -428,7 +425,8 @@ proc implementSendProcBody(sendProc: SendProc) = result.add quote do: let `delayedWriteCursor` = `prepareRequest`( - `peer`, `protocol`, `requestMethodId`, `responseMethodId`, `stream`, `timeout`, `resultIdent`) + `peer`, `protocol`, `requestMethodId`, `responseMethodId`, + `stream`, `timeout`, `resultIdent`) proc sendCallGenerator(peer, bytes: NimNode): NimNode = let @@ -452,17 +450,13 @@ proc implementSendProcBody(sendProc: SendProc) = proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = let - resultIdent = ident "result" Option = bindSym "Option" - - # XXX: Binding the int type causes instantiation failure for some reason - # Int = bindSym "int" - Int = ident "int" Peer = bindSym "Peer" EthereumNode = bindSym "EthereumNode" + Format = bindSym "SSZ" Response = bindSym "Response" - ResponseWithId = bindSym "ResponseWithId" + ResponderWithId = bindSym "ResponderWithId" perProtocolMsgId = ident"perProtocolMsgId" mount = bindSym "mount" @@ -474,9 +468,8 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = nextMsg = bindSym "nextMsg" initProtocol = bindSym "initProtocol" registerMsg = bindSym "registerMsg" + handshakeImpl = bindSym "handshakeImpl" - peer = ident "peer" - reqId = ident "reqId" stream = ident "stream" protocol = ident "protocol" response = ident "response" @@ -494,41 +487,17 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = result.PeerType = Peer result.NetworkType = EthereumNode result.SerializationFormat = Format - result.ResponseType = ResponseWithId - result.implementMsg = proc (protocol: P2PProtocol, msg: Message) = + p.useRequestIds = true + result.ResponderType = ResponderWithId + + result.implementMsg = proc (msg: Message) = var msgIdLit = newLit(msg.id) msgRecName = msg.recIdent - msgKind = msg.kind - n = msg.procDef - responseMethodId = if msg.response != nil: msg.response.id else: -1 - responseRecord = if msg.response != nil: msg.response.recIdent else: nil - msgIdent = n.name + msgIdent = msg.ident msgName = $msgIdent - hasReqIds = msgKind in {msgRequest, msgResponse} - userPragmas = n.pragma - - # variables used in the sending procs - msgRecipient = ident"msgRecipient" - rlpWriter = ident"writer" - paramsToWrite = newSeq[NimNode](0) - perPeerMsgIdVar = ident"perPeerMsgId" - - ## - ## Augment user handler - ## - if msg.userHandler != nil: - if msgKind == msgRequest: - msg.userHandler.params.insert(2, newIdentDefs(reqId, Int)) - - let peerParam = msg.userHandler.params[1][0] - let extraDefs = quote do: - let `response` = `ResponseWithId`[`responseRecord`](peer: `peerParam`, id: `reqId`) - - msg.userHandler.addPreludeDefs extraDefs - - protocol.outRecvProcs.add(msg.userHandler) + protocol = msg.protocol ## ## Implemenmt Thunk @@ -538,22 +507,25 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = else: newStmtList() - # variables used in the receiving procs - let callResolvedResponseFuture = if msgKind == msgResponse: - newCall(resolveResponseFuture, peer, msgIdLit, newCall("addr", receivedMsg), reqId) + let callResolvedResponseFuture = if msg.kind == msgResponse: + newCall(resolveResponseFuture, peer, msgIdLit, newCall("addr", receivedMsg), reqIdVar) else: newStmtList() + var userHandlerParams = @[peer] + if msg.kind == msgRequest: + userHandlerParams.add reqIdVar + let - awaitUserHandler = msg.genAwaitUserHandler(receivedMsg, peer, reqId) thunkName = ident(msgName & "_thunk") + awaitUserHandler = msg.genAwaitUserHandler(receivedMsg, userHandlerParams) var thunkProc = quote do: proc `thunkName`(`peer`: `Peer`, `stream`: `P2PStream`, - `reqId`: uint64, + `reqIdVar`: uint64, `msgContents`: `ByteStreamVar`) {.async, gcsafe.} = - var `receivedMsg` = `mount`(`SSZ`, `msgContents`, `msgRecName`) + var `receivedMsg` = `mount`(`Format`, `msgContents`, `msgRecName`) `traceMsg` `awaitUserHandler` `callResolvedResponseFuture` @@ -563,37 +535,13 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = ## ## Implement Senders and Handshake ## - var sendProc = createSendProc(msg, isRawSender = (msg.kind == msgHandshake)) + var sendProc = msg.createSendProc(isRawSender = (msg.kind == msgHandshake)) + implementSendProcBody sendProc protocol.outSendProcs.add sendProc.allDefs if msg.kind == msgHandshake: - var - rawSendProc = genSym(nskProc, msgName & "RawSend") - handshakeExchanger = createSendProc(msg, procType = nnkTemplateDef) - - handshakeExchanger.def.params[0] = newTree(nnkBracketExpr, ident("Future"), msgRecName) - - var - forwardCall = newCall(rawSendProc).appendAllParams(handshakeExchanger.def) - peerValue = forwardCall[1] - timeoutValue = msg.timeoutParam[0] - handshakeImpl = ident"handshakeImpl" - - forwardCall[1] = peer - forwardCall.del(forwardCall.len - 1) - - handshakeExchanger.def.body = quote do: - let `peer` = `peerValue` - let sendingFuture = `forwardCall` - `handshakeImpl`(`peer`, - sendingFuture, - `nextMsg`(`peer`, `msgRecName`), - `timeoutValue`) - - sendProc.def.name = rawSendProc - protocol.outSendProcs.add handshakeExchanger.def - else: - implementSendProcBody sendProc + protocol.outSendProcs.add msg.genHandshakeTemplate(sendProc.def.name, + handshakeImpl, nextMsg) protocol.outProcRegistrations.add( newCall(registerMsg, From 3b166be16658823b5b34ca66ad329d4a79224a6f Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Fri, 31 May 2019 21:36:32 +0300 Subject: [PATCH 05/18] Restore compilation in RLPx mode; More libp2p progress --- beacon_chain/eth2_network.nim | 16 ++-- beacon_chain/libp2p_backend.nim | 20 ++--- beacon_chain/libp2p_spec_backend.nim | 109 +++++++++++++++++++++------ beacon_chain/ssz.nim | 4 +- beacon_chain/version.nim | 17 ++++- 5 files changed, 124 insertions(+), 42 deletions(-) diff --git a/beacon_chain/eth2_network.nim b/beacon_chain/eth2_network.nim index 29c1348801..0d2ba95539 100644 --- a/beacon_chain/eth2_network.nim +++ b/beacon_chain/eth2_network.nim @@ -7,7 +7,7 @@ import const clientId = "Nimbus beacon node v" & fullVersionStr() -when useRLPx: +when networkBackend == rlpxBackend: import os, eth/[rlp, p2p, keys, net/nat], gossipsub_protocol, @@ -116,12 +116,14 @@ when useRLPx: node.peerPool.len else: - import - libp2p/daemon/daemonapi, - libp2p_backend - - export - libp2p_backend + import libp2p/daemon/daemonapi + + when networkBackend == libp2pSpecBackend: + import libp2p_spec_backend + export libp2p_spec_backend + else: + import libp2p_backend + export libp2p_backend type BootstrapAddr* = PeerInfo diff --git a/beacon_chain/libp2p_backend.nim b/beacon_chain/libp2p_backend.nim index 0ce4a48958..2a9837200b 100644 --- a/beacon_chain/libp2p_backend.nim +++ b/beacon_chain/libp2p_backend.nim @@ -21,7 +21,7 @@ type awaitedMessages: Table[CompressedMsgId, FutureBase] protocolStates*: seq[RootRef] - EthereumNode = Eth2Node # This alias is needed for state_helpers below + EthereumNode = Eth2Node # needed for the definitions in p2p_backends_helpers ProtocolInfoObj* = object name*: string @@ -173,7 +173,7 @@ proc p2pStreamName(MsgType: type): string = mixin msgProtocol, protocolInfo, msgId MsgType.msgProtocol.protocolInfo.messages[MsgType.msgId].libp2pProtocol -template handshakeImpl*(HandshakeTypeExpr: untyped, +template handshakeImpl(HandshakeTypeExpr: untyped, # TODO: we cannot use a type parameter above # because of the following Nim issue: # @@ -323,11 +323,11 @@ template libp2pProtocol*(name, version: string) {.pragma.} proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = var - response = ident"response" + response = ident "response" name_openStream = newTree(nnkPostfix, ident("*"), ident"openStream") - outputStream = ident"outputStream" - currentProtocolSym = ident"CurrentProtocol" - Format = ident"SSZ" + outputStream = ident "outputStream" + currentProtocolSym = ident "CurrentProtocol" + Format = ident "SSZ" Option = bindSym "Option" UntypedResponse = bindSym "UntypedResponse" Responder = bindSym "Responder" @@ -352,11 +352,11 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = registerMsg = bindSym "registerMsg" initProtocol = bindSym "initProtocol" bindSymOp = bindSym "bindSym" - msgRecipient = ident"msgRecipient" - sendTo = ident"sendTo" - writer = ident"writer" + msgRecipient = ident "msgRecipient" + sendTo = ident "sendTo" + writer = ident "writer" recordStartMemo = ident"recordStartMemo" - receivedMsg = ident"msg" + receivedMsg = ident "msg" daemon = ident "daemon" stream = ident "stream" await = ident "await" diff --git a/beacon_chain/libp2p_spec_backend.nim b/beacon_chain/libp2p_spec_backend.nim index 395a52ba46..5e055389e7 100644 --- a/beacon_chain/libp2p_spec_backend.nim +++ b/beacon_chain/libp2p_spec_backend.nim @@ -1,9 +1,12 @@ import - tables, deques, options, algorithm, std_shims/macros_shim, - chronos, chronicles, serialization, faststreams/input_stream, + tables, deques, options, algorithm, std_shims/[macros_shim, tables_shims], + ranges/ptr_arith, chronos, chronicles, serialization, faststreams/input_stream, eth/p2p/p2p_protocol_dsl, libp2p/daemon/daemonapi, ssz +export + daemonapi, p2pProtocol, ssz + const # Compression nibble NoCompression* = uint 0 @@ -11,6 +14,8 @@ const # Encoding nibble SszEncoding* = uint 1 + beaconChainProtocol = "/eth/serenity/beacon/rpc/1" + type Eth2Node* = ref object of RootObj daemon*: DaemonAPI @@ -50,7 +55,7 @@ type ServerError = 40 OutstandingRequest* = object - id*: int + id*: uint64 future*: FutureBase timeoutAt*: Moment @@ -61,7 +66,7 @@ type Peer* = ref object network*: Eth2Node id*: PeerID - lastSentMsgId*: int + lastSentMsgId*: uint64 rpcStream*: P2PStream connectionState*: ConnectionState protocolStates*: seq[RootRef] @@ -130,8 +135,9 @@ type PeerDisconnected* = object of P2PBackendError reason*: DisconnectionReason -var - gProtocols: seq[ProtocolInfo] +var gProtocols: seq[ProtocolInfo] + +template allProtocols: auto = {.gcsafe.}: gProtocols const HandshakeTimeout = FaultOrError @@ -143,8 +149,9 @@ proc `$`*(peer: Peer): string = $peer.id proc readPackedObject(stream: P2PStream, T: type): Future[T] {.async.} = await stream.transp.readExactly(addr result, sizeof result) -proc appendPackedObject*(stream: ByteStreamVar, value: auto) = - stream.append makeOpenArray(unsafeAddr value, sizeof value) +proc appendPackedObject*(stream: OutputStreamVar, value: auto) = + let valueAsBytes = cast[ptr byte](unsafeAddr(value)) + stream.append makeOpenArray(valueAsBytes, sizeof(value)) type PeerLoopExitReason = enum @@ -168,6 +175,52 @@ template raisePeerDisconnected(msg: string, r: DisconnectionReason) = e.reason = r raise e +proc handleConnectingBeaconChainPeer(daemon: DaemonAPI, stream: P2PStream) {.async, gcsafe.} + +proc init*(node: Eth2Node) {.async.} = + node.daemon = await newDaemonApi({PSGossipSub}) + node.daemon.userData = node + init node.peers + + newSeq node.protocolStates, allProtocols.len + for proto in allProtocols: + if proto.networkStateInitializer != nil: + node.protocolStates[proto.index] = proto.networkStateInitializer(node) + + await node.daemon.addHandler(@[beaconChainProtocol], handleConnectingBeaconChainPeer) + +proc init*(T: type Peer, network: Eth2Node, id: PeerID): Peer = + new result + result.id = id + result.network = network + result.awaitedMessages = initTable[CompressedMsgId, FutureBase]() + result.connectionState = Connected + newSeq result.protocolStates, allProtocols.len + for i in 0 ..< allProtocols.len: + let proto = allProtocols[i] + if proto.peerStateInitializer != nil: + result.protocolStates[i] = proto.peerStateInitializer(result) + +proc performProtocolHandshakes*(peer: Peer) {.async.} = + var subProtocolsHandshakes = newSeqOfCap[Future[void]](allProtocols.len) + for protocol in allProtocols: + if protocol.handshake != nil: + subProtocolsHandshakes.add((protocol.handshake)(peer, nil)) + + await all(subProtocolsHandshakes) + +proc getPeer*(node: Eth2Node, peerId: PeerID): Peer {.gcsafe.} = + result = node.peers.getOrDefault(peerId) + if result == nil: + result = Peer.init(node, peerId) + node.peers[peerId] = result + +proc peerFromStream(daemon: DaemonAPI, stream: P2PStream): Peer {.gcsafe.} = + Eth2Node(daemon.userData).getPeer(stream.peer) + +proc handleConnectingBeaconChainPeer(daemon: DaemonAPI, stream: P2PStream) {.async, gcsafe.} = + let peer = daemon.peerFromStream(stream) + proc disconnectAndRaise(peer: Peer, reason: DisconnectionReason, msg: string) {.async.} = @@ -244,6 +297,9 @@ proc sendMsg*(peer: Peer, data: Bytes) {.gcsafe, async.} = # sending should be retried a few times raise +proc sendMsg*[T](responder: ResponderWithId[T], data: Bytes): Future[void] = + return sendMsg(responder.peer, data) + proc nextMsg*(peer: Peer, MsgType: type): Future[MsgType] = ## This procs awaits a specific RLPx message. ## Any messages received while waiting will be dispatched to their @@ -280,7 +336,7 @@ proc registerRequest(peer: Peer, protocol: ProtocolInfo, timeout: Duration, responseFuture: FutureBase, - responseMethodId: uint16): int = + responseMethodId: uint16): uint64 = inc peer.lastSentMsgId result = peer.lastSentMsgId @@ -295,19 +351,20 @@ proc registerRequest(peer: Peer, addTimer(timeoutAt, timeoutExpired, nil) -proc resolveResponseFuture(peer: Peer, protocol: ProtocolInfo, msgId: int, msg: pointer, reqId: int) = +proc resolveResponseFuture(peer: Peer, protocol: ProtocolInfo, + methodId: int, msg: pointer, reqId: uint64) = when false: logScope: - msg = peer.dispatcher.messages[msgId].name - msgContents = peer.dispatcher.messages[msgId].printer(msg) + msg = peer.dispatcher.messages[methodId].name + msgContents = peer.dispatcher.messages[methodId].printer(msg) receivedReqId = reqId remotePeer = peer.remote template resolve(future) = - (protocol.dispatcher.messages[msgId].requestResolver)(msg, future) + (protocol.dispatcher.messages[methodId].requestResolver)(msg, future) template outstandingReqs: auto = - peer.outstandingRequests[msgId] + peer.outstandingRequests[methodId] # TODO: This is not completely sound because we are still using a global # `reqId` sequence (the problem is that we might get a response ID that @@ -395,7 +452,7 @@ template applyDecorator(p: NimNode, decorator: NimNode) = proc prepareRequest(peer: Peer, protocol: ProtocolInfo, requestMethodId, responseMethodId: uint16, - stream: ByteStreamVar, + stream: OutputStreamVar, timeout: Duration, responseFuture: FutureBase): DelayedWriteCursor = @@ -408,18 +465,23 @@ proc prepareRequest(peer: Peer, reqId: reqId, methodId: requestMethodId) +proc prepareResponse(responder: ResponderWithId, + stream: OutputStreamVar): DelayedWriteCursor = + result = stream.delayFixedSizeWrite sizeof(SpecOuterMsgHeader) + proc implementSendProcBody(sendProc: SendProc) = let msg = sendProc.msg delayedWriteCursor = ident "delayedWriteCursor" + peer = sendProc.peerParam proc preludeGenerator(stream: NimNode): NimNode = result = newStmtList() - if msg.kind == msgRequest: + case msg.kind + of msgRequest: let requestMethodId = newLit(msg.id) responseMethodId = newLit(msg.response.id) - peer = sendProc.peerParam protocol = sendProc.msg.protocol.protocolInfoVar timeout = sendProc.timeoutParam @@ -427,6 +489,11 @@ proc implementSendProcBody(sendProc: SendProc) = let `delayedWriteCursor` = `prepareRequest`( `peer`, `protocol`, `requestMethodId`, `responseMethodId`, `stream`, `timeout`, `resultIdent`) + of msgResponse: + result.add quote do: + let `delayedWriteCursor` = `prepareResponse`(`peer`, `stream`) + else: + discard proc sendCallGenerator(peer, bytes: NimNode): NimNode = let @@ -454,7 +521,7 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = Peer = bindSym "Peer" EthereumNode = bindSym "EthereumNode" - Format = bindSym "SSZ" + Format = ident "SSZ" Response = bindSym "Response" ResponderWithId = bindSym "ResponderWithId" perProtocolMsgId = ident"perProtocolMsgId" @@ -508,11 +575,11 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = newStmtList() let callResolvedResponseFuture = if msg.kind == msgResponse: - newCall(resolveResponseFuture, peer, msgIdLit, newCall("addr", receivedMsg), reqIdVar) + newCall(resolveResponseFuture, peerVar, msgIdLit, newCall("addr", receivedMsg), reqIdVar) else: newStmtList() - var userHandlerParams = @[peer] + var userHandlerParams = @[peerVar] if msg.kind == msgRequest: userHandlerParams.add reqIdVar @@ -521,7 +588,7 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = awaitUserHandler = msg.genAwaitUserHandler(receivedMsg, userHandlerParams) var thunkProc = quote do: - proc `thunkName`(`peer`: `Peer`, + proc `thunkName`(`peerVar`: `Peer`, `stream`: `P2PStream`, `reqIdVar`: uint64, `msgContents`: `ByteStreamVar`) {.async, gcsafe.} = diff --git a/beacon_chain/ssz.nim b/beacon_chain/ssz.nim index 69c686f6d2..4640bb6461 100644 --- a/beacon_chain/ssz.nim +++ b/beacon_chain/ssz.nim @@ -171,11 +171,13 @@ proc writeValue*(w: var SszWriter, obj: auto) = when obj is ValidatorIndex|BasicType: w.stream.append obj.toSSZType().toBytesSSZ + elif obj is byte|char: + w.stream.append obj elif obj is enum: w.stream.append uint64(obj).toBytesSSZ else: let memo = w.beginRecord(obj.type) - when obj is seq|array|openarray: + when obj is seq|array|openarray|string: # If you get an error here that looks like: # type mismatch: got # you just used an unsigned int for an array index thinking you'd get diff --git a/beacon_chain/version.nim b/beacon_chain/version.nim index 753cd9cff7..b4c59263c5 100644 --- a/beacon_chain/version.nim +++ b/beacon_chain/version.nim @@ -1,6 +1,16 @@ +type + NetworkBackendType* = enum + libp2pSpecBackend + libp2pNativeBackend + rlpxBackend + const - network_type {.strdefine.} = "rlpx" - useRLPx* = network_type == "rlpx" + network_type {.strdefine.} = "libp2p_spec" + + networkBackend* = when network_type == "rlpx": rlpxBackend + elif network_type == "libp2p_spec": libp2pSpecBackend + elif network_type == "libp2p_native": libp2pNativeBackend + else: {.fatal: "The 'network_type' should be one of 'libp2p_spec', 'libp2p_native' or 'rlpx'" .} const versionMajor* = 0 @@ -16,4 +26,5 @@ template versionAsStr*: string = $versionMajor & "." & $versionMinor & "." & $versionBuild proc fullVersionStr*: string = - versionAsStr & (if useRLPx: " rlpx" else: " libp2p") + versionAsStr & "_" & network_type + From 87601a5eae44fbe96fb8fc8f34ffa14322d94ec1 Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Mon, 3 Jun 2019 20:07:50 +0300 Subject: [PATCH 06/18] Share more code between the libp2p backends --- beacon_chain/eth2_network.nim | 42 +++++++- beacon_chain/libp2p_backend.nim | 155 +++++++++------------------ beacon_chain/libp2p_spec_backend.nim | 65 ++++++----- beacon_chain/ssz.nim | 3 +- beacon_chain/sync_protocol.nim | 12 +-- 5 files changed, 135 insertions(+), 142 deletions(-) diff --git a/beacon_chain/eth2_network.nim b/beacon_chain/eth2_network.nim index 0d2ba95539..4e96ca0525 100644 --- a/beacon_chain/eth2_network.nim +++ b/beacon_chain/eth2_network.nim @@ -18,6 +18,7 @@ when networkBackend == rlpxBackend: const netBackendName* = "rlpx" + IrrelevantNetwork* = UselessPeer type Eth2Node* = EthereumNode @@ -116,22 +117,30 @@ when networkBackend == rlpxBackend: node.peerPool.len else: - import libp2p/daemon/daemonapi + import + random, + libp2p/daemon/daemonapi, eth/async_utils, + ssz when networkBackend == libp2pSpecBackend: import libp2p_spec_backend export libp2p_spec_backend + + const + BreachOfProtocol* = FaultOrError + netBackendName* = "libp2p_spec" + else: import libp2p_backend export libp2p_backend + const + netBackendName* = "libp2p_native" + type BootstrapAddr* = PeerInfo Eth2NodeIdentity* = PeerInfo - const - netBackendName* = "libp2p" - proc writeValue*(writer: var JsonWriter, value: PeerID) {.inline.} = writer.writeValue value.pretty @@ -193,3 +202,28 @@ else: func peersCount*(node: Eth2Node): int = node.peers.len + proc makeMessageHandler[MsgType](msgHandler: proc(msg: MsgType)): P2PPubSubCallback = + result = proc(api: DaemonAPI, + ticket: PubsubTicket, + msg: PubSubMessage): Future[bool] {.async.} = + msgHandler SSZ.decode(msg.data, MsgType) + return true + + proc subscribe*[MsgType](node: Eth2Node, + topic: string, + msgHandler: proc(msg: MsgType)) {.async.} = + discard await node.daemon.pubsubSubscribe(topic, makeMessageHandler(msgHandler)) + + proc broadcast*(node: Eth2Node, topic: string, msg: auto) = + traceAsyncErrors node.daemon.pubsubPublish(topic, SSZ.encode(msg)) + + # TODO: + # At the moment, this is just a compatiblity shim for the existing RLPx functionality. + # The filtering is not implemented properly yet. + iterator randomPeers*(node: Eth2Node, maxPeers: int, Protocol: type): Peer = + var peers = newSeq[Peer]() + for _, peer in pairs(node.peers): peers.add peer + shuffle peers + if peers.len > maxPeers: peers.setLen(maxPeers) + for p in peers: yield p + diff --git a/beacon_chain/libp2p_backend.nim b/beacon_chain/libp2p_backend.nim index 2a9837200b..f8496e10ba 100644 --- a/beacon_chain/libp2p_backend.nim +++ b/beacon_chain/libp2p_backend.nim @@ -1,8 +1,8 @@ import - options, macros, algorithm, random, tables, + options, macros, algorithm, tables, std_shims/[macros_shim, tables_shims], chronos, chronicles, libp2p/daemon/daemonapi, faststreams/output_stream, serialization, - eth/async_utils, eth/p2p/p2p_protocol_dsl, + eth/p2p/p2p_protocol_dsl, ssz export @@ -64,11 +64,11 @@ type Disconnecting, Disconnected - UntypedResponse = object + UntypedResponder = object peer*: Peer stream*: P2PStream - Responder*[MsgType] = distinct UntypedResponse + Responder*[MsgType] = distinct UntypedResponder Bytes = seq[byte] @@ -274,7 +274,7 @@ template getRecipient(stream: P2PStream): P2PStream = stream template getRecipient(response: Responder): Peer = - UntypedResponse(response).peer + UntypedResponder(response).peer proc initProtocol(name: string, peerInit: PeerStateInitializer, @@ -308,28 +308,30 @@ proc registerProtocol(protocol: ProtocolInfo) = gProtocols[i].index = i proc getRequestProtoName(fn: NimNode): NimNode = - # `getCustomPragmaVal` doesn't work yet on regular nnkProcDef nodes - # (TODO: file as an issue) - - let pragmas = fn.pragma - if pragmas.kind == nnkPragma and pragmas.len > 0: - for pragma in pragmas: - if pragma.len > 0 and $pragma[0] == "libp2pProtocol": - return pragma[1] - - error "All stream opening procs must have the 'libp2pProtocol' pragma specified.", fn - -template libp2pProtocol*(name, version: string) {.pragma.} + when true: + return newLit("rpc/" & $fn.name) + else: + # `getCustomPragmaVal` doesn't work yet on regular nnkProcDef nodes + # (TODO: file as an issue) + let pragmas = fn.pragma + if pragmas.kind == nnkPragma and pragmas.len > 0: + for pragma in pragmas: + if pragma.len > 0 and $pragma[0] == "libp2pProtocol": + return pragma[1] + + error "All stream opening procs must have the 'libp2pProtocol' pragma specified.", fn + +proc init*[MsgType](T: type Responder[MsgType], + peer: Peer, stream: P2PStream): T = + T(UntypedResponder(peer: peer, stream: stream)) proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = var - response = ident "response" name_openStream = newTree(nnkPostfix, ident("*"), ident"openStream") outputStream = ident "outputStream" - currentProtocolSym = ident "CurrentProtocol" Format = ident "SSZ" Option = bindSym "Option" - UntypedResponse = bindSym "UntypedResponse" + UntypedResponder = bindSym "UntypedResponder" Responder = bindSym "Responder" DaemonAPI = bindSym "DaemonAPI" P2PStream = ident "P2PStream" @@ -345,6 +347,7 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = getRecipient = bindSym "getRecipient" peerFromStream = bindSym "peerFromStream" makeEth2Request = bindSym "makeEth2Request" + handshakeImpl = bindSym "handshakeImpl" sendMsg = bindSym "sendMsg" sendBytes = bindSym "sendBytes" resolveNextMsgFutures = bindSym "resolveNextMsgFutures" @@ -358,9 +361,10 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = recordStartMemo = ident"recordStartMemo" receivedMsg = ident "msg" daemon = ident "daemon" - stream = ident "stream" + streamVar = ident "stream" await = ident "await" - peerIdent = ident "peer" + + p.useRequestIds = false new result @@ -386,79 +390,42 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = ResponseRecord = if msg.response != nil: msg.response.recIdent else: nil userPragmas = n.pragma - var - # variables used in the sending procs - appendParams = newNimNode(nnkStmtList) - paramsToWrite = newSeq[NimNode](0) - - # variables used in the receiving procs - tracing = newNimNode(nnkStmtList) - - # nodes to store the user-supplied message handling proc if present - userHandlerProc: NimNode = nil - userHandlerCall: NimNode = nil - awaitUserHandler = newStmtList() - - if n.body.kind != nnkEmpty: - # This is the call to the user supplied handler. - # Here we add only the initial params, the rest will be added later. - userHandlerCall = newCall(msg.userHandler.name) - # When there is a user handler, it must be awaited in the thunk proc. - # Above, by default `awaitUserHandler` is set to a no-op statement list. - awaitUserHandler = newCall(await, userHandlerCall) - - var extraDefs: NimNode - if msgKind == msgRequest: - # Request procs need an extra param - the stream where the response - # should be written: - msg.userHandler.params.insert(1, newIdentDefs(stream, P2PStream)) - userHandlerCall.add stream - let peer = msg.userHandler.params[2][0] - extraDefs = quote do: - # Jump through some hoops to work aroung - # https://github.com/nim-lang/Nim/issues/6248 - let `response` = `Responder`[`ResponseRecord`]( - `UntypedResponse`(peer: `peer`, stream: `stream`)) - - # Resolve the Eth2Peer from the LibP2P data received in the thunk - userHandlerCall.add peerIdent - - msg.userHandler.addPreludeDefs extraDefs - protocol.outRecvProcs.add msg.userHandler - - elif msgName == "status": - #awaitUserHandler = quote do: - # `await` `handshake`(`peerIdent`, `stream`) - discard - # TODO: revisit this - - for param, paramType in n.typedParams(skip = 1): - paramsToWrite.add param + if n.body.kind != nnkEmpty and msg.kind == msgRequest: + # Request procs need an extra param - the stream where the response + # should be written: + msg.userHandler.params.insert(1, newIdentDefs(streamVar, P2PStream)) + msg.initResponderCall.add streamVar - # If there is user message handler, we'll place a call to it by - # unpacking the fields of the received message: - if userHandlerCall != nil: - userHandlerCall.add quote do: get(`receivedMsg`).`param` # newDotExpr(newCall("get", receivedMsg), param) + let awaitUserHandler = msg.genAwaitUserHandler(receivedMsg, [streamVar, peerVar]) - when tracingEnabled: - tracing = quote do: - logReceivedMsg(`stream`.peer, `receivedMsg`.get) + let tracing = when tracingEnabled: + quote do: logReceivedMsg(`streamVar`.peer, `receivedMsg`.get) + else: + newStmtList() let requestDataTimeout = newCall(milliseconds, newLit(defaultIncomingReqTimeout)) let thunkName = ident(msgName & "_thunk") var thunkProc = quote do: - proc `thunkName`(`daemon`: `DaemonAPI`, `stream`: `P2PStream`) {.async, gcsafe.} = - var `receivedMsg` = `await` readMsg(`stream`, `msgRecName`, `requestDataTimeout`) + proc `thunkName`(`daemon`: `DaemonAPI`, `streamVar`: `P2PStream`) {.async, gcsafe.} = + var `receivedMsg` = `await` readMsg(`streamVar`, `msgRecName`, `requestDataTimeout`) if `receivedMsg`.isNone: # TODO: This peer is misbehaving, perhaps we should penalize him somehow return - let `peerIdent` = `peerFromStream`(`daemon`, `stream`) + let `peerVar` = `peerFromStream`(`daemon`, `streamVar`) `tracing` `awaitUserHandler` - `resolveNextMsgFutures`(`peerIdent`, get(`receivedMsg`)) + `resolveNextMsgFutures`(`peerVar`, get(`receivedMsg`)) protocol.outRecvProcs.add thunkProc + var + # variables used in the sending procs + appendParams = newNimNode(nnkStmtList) + paramsToWrite = newSeq[NimNode](0) + + for param, paramType in n.typedParams(skip = 1): + paramsToWrite.add param + var msgSendProc = n let msgSendProcName = n.name protocol.outSendProcs.add msgSendProc @@ -498,7 +465,6 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = paramsArray = newTree(nnkBracket).appendAllParams(handshakeExchanger.def) bindSym = ident "bindSym" getAst = ident "getAst" - handshakeImpl = ident "handshakeImpl" # TODO: macros.body triggers an assertion error when the proc type is nnkMacroDef handshakeExchanger.def[6] = quote do: @@ -558,7 +524,7 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = else: quote: `sendMsg`(`msgRecipient`, `msgProto`, `msgBytes`) else: - quote: `sendBytes`(`UntypedResponse`(`sendTo`).stream, `msgBytes`) + quote: `sendBytes`(`UntypedResponder`(`sendTo`).stream, `msgBytes`) msgSendProc.body = quote do: let `msgRecipient` = `getRecipient`(`sendTo`) @@ -578,26 +544,3 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = result.implementProtocolInit = proc (p: P2PProtocol): NimNode = return newCall(initProtocol, newLit(p.name), p.peerInit, p.netInit) -proc makeMessageHandler[MsgType](msgHandler: proc(msg: MsgType)): P2PPubSubCallback = - result = proc(api: DaemonAPI, ticket: PubsubTicket, msg: PubSubMessage): Future[bool] {.async.} = - msgHandler SSZ.decode(msg.data, MsgType) - return true - -proc subscribe*[MsgType](node: EthereumNode, - topic: string, - msgHandler: proc(msg: MsgType)) {.async.} = - discard await node.daemon.pubsubSubscribe(topic, makeMessageHandler(msgHandler)) - -proc broadcast*(node: Eth2Node, topic: string, msg: auto) = - traceAsyncErrors node.daemon.pubsubPublish(topic, SSZ.encode(msg)) - -# TODO: -# At the moment, this is just a compatiblity shim for the existing RLPx functionality. -# The filtering is not implemented properly yet. -iterator randomPeers*(node: EthereumNode, maxPeers: int, Protocol: type): Peer = - var peers = newSeq[Peer]() - for _, peer in pairs(node.peers): peers.add peer - shuffle peers - if peers.len > maxPeers: peers.setLen(maxPeers) - for p in peers: yield p - diff --git a/beacon_chain/libp2p_spec_backend.nim b/beacon_chain/libp2p_spec_backend.nim index 5e055389e7..c53a39699b 100644 --- a/beacon_chain/libp2p_spec_backend.nim +++ b/beacon_chain/libp2p_spec_backend.nim @@ -5,7 +5,7 @@ import ssz export - daemonapi, p2pProtocol, ssz + daemonapi, p2pProtocol, serialization, ssz const # Compression nibble @@ -37,11 +37,11 @@ type FaultOrError CompressedMsgId = tuple - protocolIndex, msgId: int + protocolIdx, methodId: int ResponderWithId*[MsgType] = object peer*: Peer - id*: int + reqId*: uint64 Response*[MsgType] = distinct Peer @@ -125,7 +125,7 @@ type msgData: ByteStreamVar): Future[void] {.gcsafe.} MessageContentPrinter* = proc(msg: pointer): string {.gcsafe.} - NextMsgResolver* = proc(msgData: SszReader, future: FutureBase) {.gcsafe.} + NextMsgResolver* = proc(msg: pointer, future: FutureBase) {.gcsafe.} RequestResolver* = proc(msg: pointer, future: FutureBase) {.gcsafe.} Bytes = seq[byte] @@ -149,7 +149,7 @@ proc `$`*(peer: Peer): string = $peer.id proc readPackedObject(stream: P2PStream, T: type): Future[T] {.async.} = await stream.transp.readExactly(addr result, sizeof result) -proc appendPackedObject*(stream: OutputStreamVar, value: auto) = +proc appendPackedObject(stream: OutputStreamVar, value: auto) = let valueAsBytes = cast[ptr byte](unsafeAddr(value)) stream.append makeOpenArray(valueAsBytes, sizeof(value)) @@ -189,6 +189,11 @@ proc init*(node: Eth2Node) {.async.} = await node.daemon.addHandler(@[beaconChainProtocol], handleConnectingBeaconChainPeer) +proc getCompressedMsgId*(MsgType: type): CompressedMsgId = + mixin msgId, msgProtocol, protocolInfo + (protocolIdx: MsgType.msgProtocol.protocolInfo.index, + methodId: MsgType.msgId) + proc init*(T: type Peer, network: Eth2Node, id: PeerID): Peer = new result result.id = id @@ -201,6 +206,10 @@ proc init*(T: type Peer, network: Eth2Node, id: PeerID): Peer = if proto.peerStateInitializer != nil: result.protocolStates[i] = proto.peerStateInitializer(result) +proc init*[MsgName](T: type ResponderWithId[MsgName], + peer: Peer, reqId: uint64): T = + T(peer: peer, reqId: reqId) + proc performProtocolHandshakes*(peer: Peer) {.async.} = var subProtocolsHandshakes = newSeqOfCap[Future[void]](allProtocols.len) for protocol in allProtocols: @@ -301,11 +310,11 @@ proc sendMsg*[T](responder: ResponderWithId[T], data: Bytes): Future[void] = return sendMsg(responder.peer, data) proc nextMsg*(peer: Peer, MsgType: type): Future[MsgType] = - ## This procs awaits a specific RLPx message. + ## This procs awaits a specific P2P message. ## Any messages received while waiting will be dispatched to their ## respective handlers. The designated message handler will also run ## to completion before the future returned by `nextMsg` is resolved. - let wantedId = peer.perPeerMsgId(MsgType) + let wantedId = MsgType.getCompressedMsgId let f = peer.awaitedMessages[wantedId] if not f.isNil: return Future[MsgType](f) @@ -328,10 +337,6 @@ proc dispatchMessages*(peer: Peer, protocol: ProtocolInfo, stream: P2PStream): if status == Success: continue return status -proc nextMsgResolver[MsgType](msgData: ByteStreamVar, future: FutureBase) {.gcsafe.} = - var reader = msgData - Future[MsgType](future).complete reader.readRecordType(MsgType, MsgType.rlpFieldsCount > 1) - proc registerRequest(peer: Peer, protocol: ProtocolInfo, timeout: Duration, @@ -351,14 +356,13 @@ proc registerRequest(peer: Peer, addTimer(timeoutAt, timeoutExpired, nil) -proc resolveResponseFuture(peer: Peer, protocol: ProtocolInfo, +proc resolvePendingFutures(peer: Peer, protocol: ProtocolInfo, methodId: int, msg: pointer, reqId: uint64) = - when false: - logScope: - msg = peer.dispatcher.messages[methodId].name - msgContents = peer.dispatcher.messages[methodId].printer(msg) - receivedReqId = reqId - remotePeer = peer.remote + logScope: + msg = protocol.dispatcher.messages[methodId].name + msgContents = protocol.dispatcher.messages[methodId].printer(msg) + receivedReqId = reqId + remotePeer = peer.id template resolve(future) = (protocol.dispatcher.messages[methodId].requestResolver)(msg, future) @@ -366,6 +370,12 @@ proc resolveResponseFuture(peer: Peer, protocol: ProtocolInfo, template outstandingReqs: auto = peer.outstandingRequests[methodId] + let msgId = (protocolIdx: protocol.index, methodId: methodId) + if peer.awaitedMessages[msgId] != nil: + let msgInfo = protocol.dispatcher.messages[methodId] + msgInfo.nextMsgResolver(msg, peer.awaitedMessages[msgId]) + peer.awaitedMessages[msgId] = nil + # TODO: This is not completely sound because we are still using a global # `reqId` sequence (the problem is that we might get a response ID that # matches a request ID for a different type of request). To make the code @@ -407,7 +417,7 @@ proc resolveResponseFuture(peer: Peer, protocol: ProtocolInfo, inc idx - debug "late or duplicate reply for a RLPx request" + debug "late or duplicate reply for a network request" proc initProtocol(name: string, version: int, peerInit: PeerStateInitializer, @@ -529,9 +539,9 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = mount = bindSym "mount" messagePrinter = bindSym "messagePrinter" - nextMsgResolver = bindSym "nextMsgResolver" + resolveFuture = bindSym "resolveFuture" requestResolver = bindSym "requestResolver" - resolveResponseFuture = bindSym "resolveResponseFuture" + resolvePendingFutures = bindSym "resolvePendingFutures" nextMsg = bindSym "nextMsg" initProtocol = bindSym "initProtocol" registerMsg = bindSym "registerMsg" @@ -558,6 +568,9 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = p.useRequestIds = true result.ResponderType = ResponderWithId + result.afterProtocolInit = proc (p: P2PProtocol) = + p.onPeerConnected.params.add newIdentDefs(ident"handshakeStream", P2PStream) + result.implementMsg = proc (msg: Message) = var msgIdLit = newLit(msg.id) @@ -574,8 +587,10 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = else: newStmtList() - let callResolvedResponseFuture = if msg.kind == msgResponse: - newCall(resolveResponseFuture, peerVar, msgIdLit, newCall("addr", receivedMsg), reqIdVar) + let callResolvePendingFutures = if msg.kind == msgResponse: + newCall(resolvePendingFutures, + peerVar, protocol.protocolInfoVar, + msgIdLit, newCall("addr", receivedMsg), reqIdVar) else: newStmtList() @@ -595,7 +610,7 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = var `receivedMsg` = `mount`(`Format`, `msgContents`, `msgRecName`) `traceMsg` `awaitUserHandler` - `callResolvedResponseFuture` + `callResolvePendingFutures` protocol.outRecvProcs.add thunkProc @@ -618,7 +633,7 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = thunkName, newTree(nnkBracketExpr, messagePrinter, msgRecName), newTree(nnkBracketExpr, requestResolver, msgRecName), - newTree(nnkBracketExpr, nextMsgResolver, msgRecName))) + newTree(nnkBracketExpr, resolveFuture, msgRecName))) result.implementProtocolInit = proc (protocol: P2PProtocol): NimNode = return newCall(initProtocol, diff --git a/beacon_chain/ssz.nim b/beacon_chain/ssz.nim index 4640bb6461..3fd2aa1b77 100644 --- a/beacon_chain/ssz.nim +++ b/beacon_chain/ssz.nim @@ -42,7 +42,8 @@ proc init*(T: type SszReader, stream: ByteStreamVar): T = proc mount*(F: type SSZ, stream: ByteStreamVar, T: type): T = mixin readValue - init(SszReader, stream).readValue(T) + var reader = init(SszReader, stream) + reader.readValue(T) func toSSZType(x: Slot|Epoch): auto = x.uint64 func toSSZType(x: auto): auto = x diff --git a/beacon_chain/sync_protocol.nim b/beacon_chain/sync_protocol.nim index 65613b7df3..2684969689 100644 --- a/beacon_chain/sync_protocol.nim +++ b/beacon_chain/sync_protocol.nim @@ -92,7 +92,7 @@ p2pProtocol BeaconSync(version = 1, timeout = 10.seconds) if m.networkId != networkId: - await peer.disconnect(UselessPeer) + await peer.disconnect(IrrelevantNetwork) return # TODO: onPeerConnected runs unconditionally for every connected peer, but we @@ -140,7 +140,7 @@ p2pProtocol BeaconSync(version = 1, latestFinalizedRoot: Eth2Digest, latestFinalizedEpoch: Epoch, bestRoot: Eth2Digest, - bestSlot: Slot) {.libp2pProtocol("hello", "1.0.0").} + bestSlot: Slot) proc sendGoodbye(peer: Peer, reason: DisconnectionReason) @@ -163,7 +163,7 @@ p2pProtocol BeaconSync(version = 1, proc getBeaconBlockRoots( peer: Peer, fromSlot: Slot, - maxRoots: int) {.libp2pProtocol("rpc/beacon_block_roots", "1.0.0").} = + maxRoots: int) = let maxRoots = min(MaxRootsToRequest, maxRoots) var s = fromSlot var roots = newSeqOfCap[(Eth2Digest, Slot)](maxRoots) @@ -185,7 +185,7 @@ p2pProtocol BeaconSync(version = 1, slot: Slot, maxHeaders: int, skipSlots: int, - backward: uint8) {.libp2pProtocol("rpc/beacon_block_headers", "1.0.0").} = + backward: uint8) = let maxHeaders = min(MaxHeadersToRequest, maxHeaders) var headers: seq[BeaconBlockHeader] let db = peer.networkState.db @@ -235,7 +235,7 @@ p2pProtocol BeaconSync(version = 1, requestResponse: proc getAncestorBlocks( peer: Peer, - needed: openarray[FetchRecord]) {.libp2pProtocol("rpc/ancestor_blocks", "1.0.0").} = + needed: openarray[FetchRecord]) = var resp = newSeqOfCap[BeaconBlock](needed.len) let db = peer.networkState.db var neededRoots = initSet[Eth2Digest]() @@ -270,7 +270,7 @@ p2pProtocol BeaconSync(version = 1, requestResponse: proc getBeaconBlockBodies( peer: Peer, - blockRoots: openarray[Eth2Digest]) {.libp2pProtocol("rpc/beacon_block_bodies", "1.0.0").} = + blockRoots: openarray[Eth2Digest]) = # TODO: Validate blockRoots.len var bodies = newSeqOfCap[BeaconBlockBody](blockRoots.len) let db = peer.networkState.db From 877b22cfb8487fb4b8d80d586f917b8726aea5c6 Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Wed, 5 Jun 2019 05:00:07 +0300 Subject: [PATCH 07/18] Share more code between the libp2p backends --- beacon_chain/eth2_network.nim | 9 +- beacon_chain/libp2p_backend.nim | 305 +++++++----------------- beacon_chain/libp2p_backends_common.nim | 42 ++++ beacon_chain/libp2p_spec_backend.nim | 114 +++------ beacon_chain/sync_protocol.nim | 2 +- 5 files changed, 168 insertions(+), 304 deletions(-) create mode 100644 beacon_chain/libp2p_backends_common.nim diff --git a/beacon_chain/eth2_network.nim b/beacon_chain/eth2_network.nim index 4e96ca0525..d6eba1eb16 100644 --- a/beacon_chain/eth2_network.nim +++ b/beacon_chain/eth2_network.nim @@ -125,17 +125,12 @@ else: when networkBackend == libp2pSpecBackend: import libp2p_spec_backend export libp2p_spec_backend - - const - BreachOfProtocol* = FaultOrError - netBackendName* = "libp2p_spec" + const netBackendName* = "libp2p_spec" else: import libp2p_backend export libp2p_backend - - const - netBackendName* = "libp2p_native" + const netBackendName* = "libp2p_native" type BootstrapAddr* = PeerInfo diff --git a/beacon_chain/libp2p_backend.nim b/beacon_chain/libp2p_backend.nim index f8496e10ba..677ca39372 100644 --- a/beacon_chain/libp2p_backend.nim +++ b/beacon_chain/libp2p_backend.nim @@ -14,6 +14,8 @@ type peers*: Table[PeerID, Peer] protocolStates*: seq[RootRef] + EthereumNode = Eth2Node # needed for the definitions in p2p_backends_helpers + Peer* = ref object network*: Eth2Node id*: PeerID @@ -21,7 +23,31 @@ type awaitedMessages: Table[CompressedMsgId, FutureBase] protocolStates*: seq[RootRef] - EthereumNode = Eth2Node # needed for the definitions in p2p_backends_helpers + ConnectionState* = enum + None, + Connecting, + Connected, + Disconnecting, + Disconnected + + DisconnectionReason* = enum + UselessPeer + BreachOfProtocol + + UntypedResponder = object + peer*: Peer + stream*: P2PStream + + Responder*[MsgType] = distinct UntypedResponder + + MessageInfo* = object + name*: string + + # Private fields: + thunk*: ThunkProc + libp2pProtocol: string + printer*: MessageContentPrinter + nextMsgResolver*: NextMsgResolver ProtocolInfoObj* = object name*: string @@ -37,17 +63,8 @@ type ProtocolInfo* = ptr ProtocolInfoObj - MessageInfo* = object - name*: string - - # Private fields: - thunk*: ThunkProc - libp2pProtocol: string - printer*: MessageContentPrinter - nextMsgResolver*: NextMsgResolver - CompressedMsgId = tuple - protocolIndex, msgId: int + protocolIdx, methodId: int PeerStateInitializer* = proc(peer: Peer): RootRef {.gcsafe.} NetworkStateInitializer* = proc(network: EthereumNode): RootRef {.gcsafe.} @@ -57,25 +74,8 @@ type MessageContentPrinter* = proc(msg: pointer): string {.gcsafe.} NextMsgResolver* = proc(msgData: SszReader, future: FutureBase) {.gcsafe.} - ConnectionState* = enum - None, - Connecting, - Connected, - Disconnecting, - Disconnected - - UntypedResponder = object - peer*: Peer - stream*: P2PStream - - Responder*[MsgType] = distinct UntypedResponder - Bytes = seq[byte] - DisconnectionReason* = enum - UselessPeer - BreachOfProtocol - PeerDisconnected* = object of CatchableError reason*: DisconnectionReason @@ -84,34 +84,11 @@ const defaultOutgoingReqTimeout = 10000 HandshakeTimeout = BreachOfProtocol -var - gProtocols: seq[ProtocolInfo] - -# The variables above are immutable RTTI information. We need to tell -# Nim to not consider them GcSafe violations: -template allProtocols: auto = {.gcsafe.}: gProtocols - -proc `$`*(peer: Peer): string = $peer.id + IrrelevantNetwork* = UselessPeer -proc disconnect*(peer: Peer, reason: DisconnectionReason, notifyOtherPeer = false) {.async.} = - # TODO: How should we notify the other peer? - if peer.connectionState notin {Disconnecting, Disconnected}: - peer.connectionState = Disconnecting - await peer.network.daemon.disconnect(peer.id) - peer.connectionState = Disconnected - peer.network.peers.del(peer.id) - -template raisePeerDisconnected(msg: string, r: DisconnectionReason) = - var e = newException(PeerDisconnected, msg) - e.reason = r - raise e - -proc disconnectAndRaise(peer: Peer, - reason: DisconnectionReason, - msg: string) {.async.} = - let r = reason - await peer.disconnect(reason) - raisePeerDisconnected(msg, reason) +include libp2p_backends_common +include eth/p2p/p2p_backends_helpers +include eth/p2p/p2p_tracing proc init*(node: Eth2Node) {.async.} = node.daemon = await newDaemonApi({PSGossipSub}) @@ -127,9 +104,6 @@ proc init*(node: Eth2Node) {.async.} = if msg.libp2pProtocol.len > 0: await node.daemon.addHandler(@[msg.libp2pProtocol], msg.thunk) -include eth/p2p/p2p_backends_helpers -include eth/p2p/p2p_tracing - proc readMsg(stream: P2PStream, MsgType: type, timeout = 10.seconds): Future[Option[MsgType]] {.async.} = var timeout = sleepAsync timeout @@ -212,24 +186,6 @@ template handshakeImpl(HandshakeTypeExpr: untyped, asyncStep(stream) -proc getCompressedMsgId(MsgType: type): CompressedMsgId = - mixin msgProtocol, protocolInfo, msgId - (protocolIndex: MsgType.msgProtocol.protocolInfo.index, msgId: MsgType.msgId) - -proc nextMsg*(peer: Peer, MsgType: type): Future[MsgType] = - ## This procs awaits a specific P2P message. - ## Any messages received while waiting will be dispatched to their - ## respective handlers. The designated message handler will also run - ## to completion before the future returned by `nextMsg` is resolved. - mixin msgProtocol, protocolInfo, msgId - let awaitedMsgId = getCompressedMsgId(MsgType) - let f = getOrDefault(peer.awaitedMessages, awaitedMsgId) - if not f.isNil: - return Future[MsgType](f) - - newFuture result - peer.awaitedMessages[awaitedMsgId] = result - proc resolveNextMsgFutures(peer: Peer, msg: auto) = type MsgType = type(msg) let msgId = getCompressedMsgId(MsgType) @@ -284,6 +240,13 @@ proc initProtocol(name: string, result.peerStateInitializer = peerInit result.networkStateInitializer = networkInit +proc registerProtocol(protocol: ProtocolInfo) = + # TODO: This can be done at compile-time in the future + let pos = lowerBound(gProtocols, protocol) + gProtocols.insert(protocol, pos) + for i in 0 ..< gProtocols.len: + gProtocols[i].index = i + proc setEventHandlers(p: ProtocolInfo, handshake: HandshakeStep, disconnectHandler: DisconnectionHandler) = @@ -300,13 +263,6 @@ proc registerMsg(protocol: ProtocolInfo, libp2pProtocol: libp2pProtocol, printer: printer) -proc registerProtocol(protocol: ProtocolInfo) = - # TODO: This can be done at compile-time in the future - let pos = lowerBound(gProtocols, protocol) - gProtocols.insert(protocol, pos) - for i in 0 ..< gProtocols.len: - gProtocols[i].index = i - proc getRequestProtoName(fn: NimNode): NimNode = when true: return newLit("rpc/" & $fn.name) @@ -325,42 +281,52 @@ proc init*[MsgType](T: type Responder[MsgType], peer: Peer, stream: P2PStream): T = T(UntypedResponder(peer: peer, stream: stream)) +proc implementSendProcBody(sendProc: SendProc) = + let + msg = sendProc.msg + peer = sendProc.peerParam + timeout = sendProc.timeoutParam + ResponseRecord = if msg.response != nil: msg.response.recIdent else: nil + UntypedResponder = bindSym "UntypedResponder" + sendMsg = bindSym "sendMsg" + sendBytes = bindSym "sendBytes" + makeEth2Request = bindSym "makeEth2Request" + + proc sendCallGenerator(peer, bytes: NimNode): NimNode = + if msg.kind != msgResponse: + let msgProto = getRequestProtoName(msg.procDef) + case msg.kind + of msgRequest: + let timeout = msg.timeoutParam[0] + quote: `makeEth2Request`(`peer`, `msgProto`, `bytes`, + `ResponseRecord`, `timeout`) + of msgHandshake: + quote: `sendBytes`(`peer`, `bytes`) + else: + quote: `sendMsg`(`peer`, `msgProto`, `bytes`) + else: + quote: `sendBytes`(`UntypedResponder`(`peer`).stream, `bytes`) + + sendProc.useStandardBody(nil, sendCallGenerator) + proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = var - name_openStream = newTree(nnkPostfix, ident("*"), ident"openStream") - outputStream = ident "outputStream" Format = ident "SSZ" - Option = bindSym "Option" - UntypedResponder = bindSym "UntypedResponder" Responder = bindSym "Responder" DaemonAPI = bindSym "DaemonAPI" P2PStream = ident "P2PStream" - # XXX: Binding the int type causes instantiation failure for some reason - # Int = bindSym "int" - Int = ident "int" - Void = ident "void" Peer = bindSym "Peer" Eth2Node = bindSym "Eth2Node" - writeField = bindSym "writeField" - getOutput = bindSym "getOutput" messagePrinter = bindSym "messagePrinter" - getRecipient = bindSym "getRecipient" peerFromStream = bindSym "peerFromStream" - makeEth2Request = bindSym "makeEth2Request" handshakeImpl = bindSym "handshakeImpl" - sendMsg = bindSym "sendMsg" - sendBytes = bindSym "sendBytes" resolveNextMsgFutures = bindSym "resolveNextMsgFutures" milliseconds = bindSym "milliseconds" registerMsg = bindSym "registerMsg" initProtocol = bindSym "initProtocol" bindSymOp = bindSym "bindSym" - msgRecipient = ident "msgRecipient" - sendTo = ident "sendTo" - writer = ident "writer" - recordStartMemo = ident"recordStartMemo" receivedMsg = ident "msg" - daemon = ident "daemon" + daemonVar = ident "daemon" streamVar = ident "stream" await = ident "await" @@ -380,94 +346,53 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = result.implementMsg = proc (msg: Message) = let - n = msg.procDef protocol = msg.protocol - msgId = newLit(msg.id) - msgIdent = n.name - msgName = $msgIdent - msgKind = msg.kind + msgName = $msg.ident msgRecName = msg.recIdent - ResponseRecord = if msg.response != nil: msg.response.recIdent else: nil - userPragmas = n.pragma - if n.body.kind != nnkEmpty and msg.kind == msgRequest: + if msg.procDef.body.kind != nnkEmpty and msg.kind == msgRequest: # Request procs need an extra param - the stream where the response # should be written: - msg.userHandler.params.insert(1, newIdentDefs(streamVar, P2PStream)) + msg.userHandler.params.insert(2, newIdentDefs(streamVar, P2PStream)) msg.initResponderCall.add streamVar - let awaitUserHandler = msg.genAwaitUserHandler(receivedMsg, [streamVar, peerVar]) + let awaitUserHandler = msg.genAwaitUserHandler(newCall("get", receivedMsg), [peerVar, streamVar]) let tracing = when tracingEnabled: quote do: logReceivedMsg(`streamVar`.peer, `receivedMsg`.get) else: newStmtList() - let requestDataTimeout = newCall(milliseconds, newLit(defaultIncomingReqTimeout)) - let thunkName = ident(msgName & "_thunk") - var thunkProc = quote do: - proc `thunkName`(`daemon`: `DaemonAPI`, `streamVar`: `P2PStream`) {.async, gcsafe.} = + let + requestDataTimeout = newCall(milliseconds, newLit(defaultIncomingReqTimeout)) + thunkName = ident(msgName & "_thunk") + + msg.defineThunk quote do: + proc `thunkName`(`daemonVar`: `DaemonAPI`, `streamVar`: `P2PStream`) {.async, gcsafe.} = var `receivedMsg` = `await` readMsg(`streamVar`, `msgRecName`, `requestDataTimeout`) if `receivedMsg`.isNone: # TODO: This peer is misbehaving, perhaps we should penalize him somehow return - let `peerVar` = `peerFromStream`(`daemon`, `streamVar`) + let `peerVar` = `peerFromStream`(`daemonVar`, `streamVar`) `tracing` `awaitUserHandler` `resolveNextMsgFutures`(`peerVar`, get(`receivedMsg`)) - protocol.outRecvProcs.add thunkProc - - var - # variables used in the sending procs - appendParams = newNimNode(nnkStmtList) - paramsToWrite = newSeq[NimNode](0) - - for param, paramType in n.typedParams(skip = 1): - paramsToWrite.add param - - var msgSendProc = n - let msgSendProcName = n.name - protocol.outSendProcs.add msgSendProc - - # TODO: check that the first param has the correct type - msgSendProc.params[1][0] = sendTo - msgSendProc.addPragma ident"gcsafe" - - # Add a timeout parameter for all request procs - case msgKind - of msgRequest: - # Add a timeout parameter for all request procs - msgSendProc.params.add msg.timeoutParam - of msgResponse: - # A response proc must be called with a response object that originates - # from a certain request. Here we change the Peer parameter at position - # 1 to the correct strongly-typed ResponseType. The incoming procs still - # gets the normal Peer paramter. - let ResponseType = newTree(nnkBracketExpr, Responder, msgRecName) - msgSendProc.params[1][1] = ResponseType - protocol.outSendProcs.add quote do: - template send*(r: `ResponseType`, args: varargs[untyped]): auto = - `msgSendProcName`(r, args) - else: discard - - # We change the return type of the sending proc to a Future. - # If this is a request proc, the future will return the response record. - let rt = if msgKind != msgRequest: Void - else: newTree(nnkBracketExpr, Option, ResponseRecord) - msgSendProc.params[0] = newTree(nnkBracketExpr, ident("Future"), rt) - - if msgKind == msgHandshake: + ## + ## Implement Senders and Handshake + ## + var sendProc = msg.createSendProc(isRawSender = (msg.kind == msgHandshake)) + implementSendProcBody sendProc + + if msg.kind == msgHandshake: var - rawSendProc = msgName & "RawSend" - handshakeTypeName = $msgRecName + rawSendProc = newLit($sendProc.def.name) handshakeExchanger = msg.createSendProc(nnkMacroDef) paramsArray = newTree(nnkBracket).appendAllParams(handshakeExchanger.def) bindSym = ident "bindSym" getAst = ident "getAst" - # TODO: macros.body triggers an assertion error when the proc type is nnkMacroDef - handshakeExchanger.def[6] = quote do: + handshakeExchanger.setBody quote do: let stream = ident"handshakeStream" rawSendProc = `bindSymOp` `rawSendProc` @@ -481,64 +406,14 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = return `getAst`(`handshakeImpl`(`msgRecName`, peer, stream, lazySendCall, timeout)) - protocol.outSendProcs.add handshakeExchanger.def - - msgSendProc.params[1][1] = P2PStream - msgSendProc.name = ident rawSendProc - else: - # Make the send proc public - msgSendProc.name = msg.identWithExportMarker - - let initWriter = quote do: - var `outputStream` = init OutputStream - var `writer` = init(WriterType(`Format`), `outputStream`) - var `recordStartMemo` = beginRecord(`writer`, `msgRecName`) - - for param in paramsToWrite: - appendParams.add newCall(writeField, writer, newLit($param), param) - - when tracingEnabled: - appendParams.add logSentMsgFields(msgRecipient, protocol, msgName, paramsToWrite) - - let msgBytes = ident"msgBytes" - let finalizeRequest = quote do: - endRecord(`writer`, `recordStartMemo`) - let `msgBytes` = `getOutput`(`outputStream`) - - var msgProto = newLit("") - let sendCall = - if msgKind != msgResponse: - msgProto = getRequestProtoName(n) - - when false: - var openStreamProc = n.copyNimTree - var openStreamProc.name = name_openStream - openStreamProc.params.insert 1, newIdentDefs(ident"T", msgRecName) - - if msgKind == msgRequest: - let timeout = msg.timeoutParam[0] - quote: `makeEth2Request`(`msgRecipient`, `msgProto`, `msgBytes`, - `ResponseRecord`, `timeout`) - elif msgId.intVal == 0: - quote: `sendBytes`(`sendTo`, `msgBytes`) - else: - quote: `sendMsg`(`msgRecipient`, `msgProto`, `msgBytes`) - else: - quote: `sendBytes`(`UntypedResponder`(`sendTo`).stream, `msgBytes`) - - msgSendProc.body = quote do: - let `msgRecipient` = `getRecipient`(`sendTo`) - `initWriter` - `appendParams` - `finalizeRequest` - return `sendCall` + sendProc.def.params[1][1] = P2PStream protocol.outProcRegistrations.add( newCall(registerMsg, protocol.protocolInfoVar, newLit(msgName), thunkName, - msgProto, + getRequestProtoName(msg.procDef), newTree(nnkBracketExpr, messagePrinter, msgRecName))) result.implementProtocolInit = proc (p: P2PProtocol): NimNode = diff --git a/beacon_chain/libp2p_backends_common.nim b/beacon_chain/libp2p_backends_common.nim new file mode 100644 index 0000000000..24a4e6919d --- /dev/null +++ b/beacon_chain/libp2p_backends_common.nim @@ -0,0 +1,42 @@ +# included from libp2p_backend + +proc `$`*(peer: Peer): string = $peer.id + +proc disconnect*(peer: Peer, reason: DisconnectionReason, notifyOtherPeer = false) {.async.} = + # TODO: How should we notify the other peer? + if peer.connectionState notin {Disconnecting, Disconnected}: + peer.connectionState = Disconnecting + await peer.network.daemon.disconnect(peer.id) + peer.connectionState = Disconnected + peer.network.peers.del(peer.id) + +template raisePeerDisconnected(msg: string, r: DisconnectionReason) = + var e = newException(PeerDisconnected, msg) + e.reason = r + raise e + +proc disconnectAndRaise(peer: Peer, + reason: DisconnectionReason, + msg: string) {.async.} = + let r = reason + await peer.disconnect(r) + raisePeerDisconnected(msg, r) + +proc getCompressedMsgId*(MsgType: type): CompressedMsgId = + mixin msgId, msgProtocol, protocolInfo + (protocolIdx: MsgType.msgProtocol.protocolInfo.index, + methodId: MsgType.msgId) + +proc nextMsg*(peer: Peer, MsgType: type): Future[MsgType] = + ## This procs awaits a specific P2P message. + ## Any messages received while waiting will be dispatched to their + ## respective handlers. The designated message handler will also run + ## to completion before the future returned by `nextMsg` is resolved. + let awaitedMsgId = getCompressedMsgId(MsgType) + let f = getOrDefault(peer.awaitedMessages, awaitedMsgId) + if not f.isNil: + return Future[MsgType](f) + + initFuture result + peer.awaitedMessages[awaitedMsgId] = result + diff --git a/beacon_chain/libp2p_spec_backend.nim b/beacon_chain/libp2p_spec_backend.nim index c53a39699b..3d57db53b2 100644 --- a/beacon_chain/libp2p_spec_backend.nim +++ b/beacon_chain/libp2p_spec_backend.nim @@ -24,6 +24,17 @@ type EthereumNode = Eth2Node # needed for the definitions in p2p_backends_helpers + Peer* = ref object + network*: Eth2Node + id*: PeerID + lastSentMsgId*: uint64 + rpcStream*: P2PStream + connectionState*: ConnectionState + awaitedMessages: Table[CompressedMsgId, FutureBase] + outstandingRequests*: seq[Deque[OutstandingRequest]] + protocolStates*: seq[RootRef] + maxInactivityAllowed: Duration + ConnectionState* = enum None, Connecting, @@ -63,24 +74,12 @@ type stream*: P2PStream protocolInfo*: ProtocolInfo - Peer* = ref object - network*: Eth2Node - id*: PeerID - lastSentMsgId*: uint64 - rpcStream*: P2PStream - connectionState*: ConnectionState - protocolStates*: seq[RootRef] - maxInactivityAllowed: Duration - awaitedMessages: Table[CompressedMsgId, FutureBase] - outstandingRequests*: seq[Deque[OutstandingRequest]] - MessageInfo* = object id*: int name*: string # Private fields: thunk*: ThunkProc - libp2pProtocol: string printer*: MessageContentPrinter nextMsgResolver*: NextMsgResolver requestResolver*: RequestResolver @@ -135,16 +134,18 @@ type PeerDisconnected* = object of P2PBackendError reason*: DisconnectionReason -var gProtocols: seq[ProtocolInfo] - -template allProtocols: auto = {.gcsafe.}: gProtocols + PeerLoopExitReason = enum + Success + UnsupportedCompression + UnsupportedEncoding + ProtocolViolation + InactivePeer + InternalError const HandshakeTimeout = FaultOrError - # TODO: this doesn't seem right. - # We should lobby for more disconnection reasons. - -proc `$`*(peer: Peer): string = $peer.id + BreachOfProtocol* = FaultOrError + # TODO: We should lobby for more disconnection reasons. proc readPackedObject(stream: P2PStream, T: type): Future[T] {.async.} = await stream.transp.readExactly(addr result, sizeof result) @@ -153,27 +154,9 @@ proc appendPackedObject(stream: OutputStreamVar, value: auto) = let valueAsBytes = cast[ptr byte](unsafeAddr(value)) stream.append makeOpenArray(valueAsBytes, sizeof(value)) -type - PeerLoopExitReason = enum - Success - UnsupportedCompression - UnsupportedEncoding - ProtocolViolation - InactivePeer - InternalError - -proc disconnect*(peer: Peer, reason: DisconnectionReason, notifyOtherPeer = false) {.async.} = - # TODO: How should we notify the other peer? - if peer.connectionState notin {Disconnecting, Disconnected}: - peer.connectionState = Disconnecting - await peer.network.daemon.disconnect(peer.id) - peer.connectionState = Disconnected - peer.network.peers.del(peer.id) - -template raisePeerDisconnected(msg: string, r: DisconnectionReason) = - var e = newException(PeerDisconnected, msg) - e.reason = r - raise e +include libp2p_backends_common +include eth/p2p/p2p_backends_helpers +include eth/p2p/p2p_tracing proc handleConnectingBeaconChainPeer(daemon: DaemonAPI, stream: P2PStream) {.async, gcsafe.} @@ -189,11 +172,6 @@ proc init*(node: Eth2Node) {.async.} = await node.daemon.addHandler(@[beaconChainProtocol], handleConnectingBeaconChainPeer) -proc getCompressedMsgId*(MsgType: type): CompressedMsgId = - mixin msgId, msgProtocol, protocolInfo - (protocolIdx: MsgType.msgProtocol.protocolInfo.index, - methodId: MsgType.msgId) - proc init*(T: type Peer, network: Eth2Node, id: PeerID): Peer = new result result.id = id @@ -230,16 +208,6 @@ proc peerFromStream(daemon: DaemonAPI, stream: P2PStream): Peer {.gcsafe.} = proc handleConnectingBeaconChainPeer(daemon: DaemonAPI, stream: P2PStream) {.async, gcsafe.} = let peer = daemon.peerFromStream(stream) -proc disconnectAndRaise(peer: Peer, - reason: DisconnectionReason, - msg: string) {.async.} = - let r = reason - await peer.disconnect(r) - raisePeerDisconnected(msg, r) - -include eth/p2p/p2p_backends_helpers -include eth/p2p/p2p_tracing - proc accepts(d: Dispatcher, methodId: uint16): bool = methodId.int < d.messages.len @@ -309,19 +277,6 @@ proc sendMsg*(peer: Peer, data: Bytes) {.gcsafe, async.} = proc sendMsg*[T](responder: ResponderWithId[T], data: Bytes): Future[void] = return sendMsg(responder.peer, data) -proc nextMsg*(peer: Peer, MsgType: type): Future[MsgType] = - ## This procs awaits a specific P2P message. - ## Any messages received while waiting will be dispatched to their - ## respective handlers. The designated message handler will also run - ## to completion before the future returned by `nextMsg` is resolved. - let wantedId = MsgType.getCompressedMsgId - let f = peer.awaitedMessages[wantedId] - if not f.isNil: - return Future[MsgType](f) - - initFuture result - peer.awaitedMessages[wantedId] = result - proc dispatchMessages*(peer: Peer, protocol: ProtocolInfo, stream: P2PStream): Future[PeerLoopExitReason] {.async.} = while true: @@ -428,12 +383,6 @@ proc initProtocol(name: string, version: int, result.peerStateInitializer = peerInit result.networkStateInitializer = networkInit -proc setEventHandlers(p: ProtocolInfo, - handshake: HandshakeStep, - disconnectHandler: DisconnectionHandler) = - p.handshake = handshake - p.disconnectHandler = disconnectHandler - proc registerProtocol(protocol: ProtocolInfo) = # TODO: This can be done at compile-time in the future let pos = lowerBound(gProtocols, protocol) @@ -441,6 +390,12 @@ proc registerProtocol(protocol: ProtocolInfo) = for i in 0 ..< gProtocols.len: gProtocols[i].index = i +proc setEventHandlers(p: ProtocolInfo, + handshake: HandshakeStep, + disconnectHandler: DisconnectionHandler) = + p.handshake = handshake + p.disconnectHandler = disconnectHandler + proc registerMsg(protocol: ProtocolInfo, id: int, name: string, thunk: ThunkProc, @@ -523,7 +478,7 @@ proc implementSendProcBody(sendProc: SendProc) = # `sendMsg` call. quote: return `sendCall` - sendProc.implementBody(preludeGenerator, sendCallGenerator) + sendProc.useStandardBody(preludeGenerator, sendCallGenerator) proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = let @@ -566,6 +521,7 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = result.SerializationFormat = Format p.useRequestIds = true + result.ReqIdType = ident "uint64" result.ResponderType = ResponderWithId result.afterProtocolInit = proc (p: P2PProtocol) = @@ -602,7 +558,7 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = thunkName = ident(msgName & "_thunk") awaitUserHandler = msg.genAwaitUserHandler(receivedMsg, userHandlerParams) - var thunkProc = quote do: + msg.defineThunk quote do: proc `thunkName`(`peerVar`: `Peer`, `stream`: `P2PStream`, `reqIdVar`: uint64, @@ -612,18 +568,14 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = `awaitUserHandler` `callResolvePendingFutures` - protocol.outRecvProcs.add thunkProc - ## ## Implement Senders and Handshake ## var sendProc = msg.createSendProc(isRawSender = (msg.kind == msgHandshake)) implementSendProcBody sendProc - protocol.outSendProcs.add sendProc.allDefs if msg.kind == msgHandshake: - protocol.outSendProcs.add msg.genHandshakeTemplate(sendProc.def.name, - handshakeImpl, nextMsg) + discard msg.createHandshakeTemplate(sendProc.def.name, handshakeImpl, nextMsg) protocol.outProcRegistrations.add( newCall(registerMsg, diff --git a/beacon_chain/sync_protocol.nim b/beacon_chain/sync_protocol.nim index 2684969689..9ab26f2fb0 100644 --- a/beacon_chain/sync_protocol.nim +++ b/beacon_chain/sync_protocol.nim @@ -74,7 +74,7 @@ p2pProtocol BeaconSync(version = 1, shortName = "bcs", networkState = BeaconSyncState): - onPeerConnected do(peer: Peer): + onPeerConnected do (peer: Peer): let protocolVersion = 1 # TODO: Spec doesn't specify this yet node = peer.networkState.node From 12e9367f78bfd8a47739974334759d2448eb8d4f Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Tue, 11 Jun 2019 02:20:18 +0300 Subject: [PATCH 08/18] Improved error handling; Simple test case for connecting 2 peers --- beacon_chain/conf.nim | 2 +- beacon_chain/eth2_network.nim | 2 +- beacon_chain/libp2p_backend.nim | 77 +++++++++++++++---------- beacon_chain/libp2p_backends_common.nim | 17 ++++++ beacon_chain/libp2p_spec_backend.nim | 20 ++++--- tests/test_peer_connection.nim | 24 ++++++++ 6 files changed, 99 insertions(+), 43 deletions(-) create mode 100644 tests/test_peer_connection.nim diff --git a/beacon_chain/conf.nim b/beacon_chain/conf.nim index 7e9311a4ce..403c53e251 100644 --- a/beacon_chain/conf.nim +++ b/beacon_chain/conf.nim @@ -4,7 +4,7 @@ import spec/[crypto, datatypes], time, version export - defs + defs, enabledLogLevel const DEFAULT_NETWORK* {.strdefine.} = "testnet0" diff --git a/beacon_chain/eth2_network.nim b/beacon_chain/eth2_network.nim index d6eba1eb16..7f660b4352 100644 --- a/beacon_chain/eth2_network.nim +++ b/beacon_chain/eth2_network.nim @@ -183,7 +183,7 @@ else: try: await node.daemon.connect(bootstrapNode.peer, bootstrapNode.addresses) let peer = node.getPeer(bootstrapNode.peer) - await peer.performProtocolHandshakes() + await initializeConnection(peer) except PeerDisconnected: error "Failed to connect to bootstrap node", node = bootstrapNode diff --git a/beacon_chain/libp2p_backend.nim b/beacon_chain/libp2p_backend.nim index 677ca39372..ac2bf3efc6 100644 --- a/beacon_chain/libp2p_backend.nim +++ b/beacon_chain/libp2p_backend.nim @@ -33,6 +33,7 @@ type DisconnectionReason* = enum UselessPeer BreachOfProtocol + FaultOrError UntypedResponder = object peer*: Peer @@ -105,17 +106,16 @@ proc init*(node: Eth2Node) {.async.} = await node.daemon.addHandler(@[msg.libp2pProtocol], msg.thunk) proc readMsg(stream: P2PStream, MsgType: type, - timeout = 10.seconds): Future[Option[MsgType]] {.async.} = - var timeout = sleepAsync timeout + deadline: Future[void]): Future[Option[MsgType]] {.async.} = var sizePrefix: uint32 var readSizePrefix = stream.transp.readExactly(addr sizePrefix, sizeof(sizePrefix)) - await readSizePrefix or timeout + await readSizePrefix or deadline if not readSizePrefix.finished: return var msgBytes = newSeq[byte](sizePrefix.int + sizeof(sizePrefix)) copyMem(addr msgBytes[0], addr sizePrefix, sizeof(sizePrefix)) var readBody = stream.transp.readExactly(addr msgBytes[sizeof(sizePrefix)], sizePrefix.int) - await readBody or timeout + await readBody or deadline if not readBody.finished: return let decoded = SSZ.decode(msgBytes, MsgType) @@ -137,11 +137,22 @@ proc sendBytes(stream: P2PStream, bytes: Bytes) {.async.} = proc makeEth2Request(peer: Peer, protocolId: string, requestBytes: Bytes, ResponseMsg: type, timeout = 10.seconds): Future[Option[ResponseMsg]] {.async.} = - var stream = await peer.network.daemon.openStream(peer.id, @[protocolId]) - # TODO how does openStream fail? Set a timeout here and handle it + var deadline = sleepAsync timeout + + # Open a new LibP2P stream + var streamFut = peer.network.daemon.openStream(peer.id, @[protocolId]) + await streamFut or deadline + if not streamFut.finished: + return none(ResponseMsg) + + # Send the request + let stream = streamFut.read let sent = await stream.transp.write(requestBytes) - # TODO: Should I check that `sent` is equal to the desired number of bytes - return await stream.readMsg(ResponseMsg, timeout) + if sent != requestBytes: + await disconnectAndRaise(peer, FaultOrError, "Incomplete send") + + # Read the response + return await stream.readMsg(ResponseMsg, deadline) proc p2pStreamName(MsgType: type): string = mixin msgProtocol, protocolInfo, msgId @@ -167,22 +178,31 @@ template handshakeImpl(HandshakeTypeExpr: untyped, type HandshakeType = type(HandshakeTypeExpr) proc asyncStep(stream: P2PStream): Future[HandshakeType] {.async.} = + let deadline = sleepAsync timeout var stream = stream if stream == nil: - stream = await openStream(peer.network.daemon, peer.id, - @[p2pStreamName(HandshakeType)], - # TODO openStream should accept Duration - int milliseconds(timeout)) - - # Please pay attention that `lazySendCall` is evaluated lazily here. - # For this reason `handshakeImpl` must remain a template. - await lazySendCall - - let response = await readMsg(stream, HandshakeType, timeout) - if response.isSome: - return response.get - else: - await disconnectAndRaise(peer, BreachOfProtocol, "Handshake not completed in time") + try: stream = await openStream(peer.network.daemon, peer.id, + @[p2pStreamName(HandshakeType)], + # TODO openStream should accept Duration + int milliseconds(timeout)) + except CatchableError: + const errMsg = "Failed to open LIBP2P stream" + debug errMsg, + stream = p2pStreamName(HandshakeType), + err = getCurrentExceptionMsg() + await disconnectAndRaise(peer, FaultOrError, errMsg) + + try: + # Please pay attention that `lazySendCall` is evaluated lazily here. + # For this reason `handshakeImpl` must remain a template. + await lazySendCall + let response = await readMsg(stream, HandshakeType, deadline) + if response.isSome: + return response.get + else: + await disconnectAndRaise(peer, BreachOfProtocol, "Handshake not completed in time") + except CatchableError: + await reraiseAsPeerDisconnected(peer, "Failure during handshake") asyncStep(stream) @@ -213,15 +233,6 @@ proc performProtocolHandshakes*(peer: Peer) {.async.} = await all(subProtocolsHandshakes) -proc getPeer*(node: Eth2Node, peerId: PeerID): Peer = - result = node.peers.getOrDefault(peerId) - if result == nil: - result = Peer.init(node, peerId) - node.peers[peerId] = result - -proc peerFromStream(daemon: DaemonAPI, stream: P2PStream): Peer = - Eth2Node(daemon.userData).getPeer(stream.peer) - template getRecipient(peer: Peer): Peer = peer @@ -328,6 +339,7 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = receivedMsg = ident "msg" daemonVar = ident "daemon" streamVar = ident "stream" + deadlineVar = ident "deadline" await = ident "await" p.useRequestIds = false @@ -369,7 +381,8 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = msg.defineThunk quote do: proc `thunkName`(`daemonVar`: `DaemonAPI`, `streamVar`: `P2PStream`) {.async, gcsafe.} = - var `receivedMsg` = `await` readMsg(`streamVar`, `msgRecName`, `requestDataTimeout`) + var `deadlineVar` = sleepAsync `requestDataTimeout` + var `receivedMsg` = `await` readMsg(`streamVar`, `msgRecName`, `deadlineVar`) if `receivedMsg`.isNone: # TODO: This peer is misbehaving, perhaps we should penalize him somehow return diff --git a/beacon_chain/libp2p_backends_common.nim b/beacon_chain/libp2p_backends_common.nim index 24a4e6919d..24318727c8 100644 --- a/beacon_chain/libp2p_backends_common.nim +++ b/beacon_chain/libp2p_backends_common.nim @@ -2,6 +2,17 @@ proc `$`*(peer: Peer): string = $peer.id +proc init*(T: type Peer, network: Eth2Node, id: PeerID): Peer {.gcsafe.} + +proc getPeer*(node: Eth2Node, peerId: PeerID): Peer {.gcsafe.} = + result = node.peers.getOrDefault(peerId) + if result == nil: + result = Peer.init(node, peerId) + node.peers[peerId] = result + +proc peerFromStream(daemon: DaemonAPI, stream: P2PStream): Peer {.gcsafe.} = + Eth2Node(daemon.userData).getPeer(stream.peer) + proc disconnect*(peer: Peer, reason: DisconnectionReason, notifyOtherPeer = false) {.async.} = # TODO: How should we notify the other peer? if peer.connectionState notin {Disconnecting, Disconnected}: @@ -22,6 +33,12 @@ proc disconnectAndRaise(peer: Peer, await peer.disconnect(r) raisePeerDisconnected(msg, r) +template reraiseAsPeerDisconnected(peer: Peer, errMsgExpr: static string, + reason = FaultOrError): auto = + const errMsg = errMsgExpr + debug errMsg, err = getCurrentExceptionMsg() + disconnectAndRaise(peer, reason, errMsg) + proc getCompressedMsgId*(MsgType: type): CompressedMsgId = mixin msgId, msgProtocol, protocolInfo (protocolIdx: MsgType.msgProtocol.protocolInfo.index, diff --git a/beacon_chain/libp2p_spec_backend.nim b/beacon_chain/libp2p_spec_backend.nim index 3d57db53b2..eca716db79 100644 --- a/beacon_chain/libp2p_spec_backend.nim +++ b/beacon_chain/libp2p_spec_backend.nim @@ -192,21 +192,23 @@ proc performProtocolHandshakes*(peer: Peer) {.async.} = var subProtocolsHandshakes = newSeqOfCap[Future[void]](allProtocols.len) for protocol in allProtocols: if protocol.handshake != nil: - subProtocolsHandshakes.add((protocol.handshake)(peer, nil)) + subProtocolsHandshakes.add((protocol.handshake)(peer, peer.rpcStream)) await all(subProtocolsHandshakes) + debug "All protocols initialized", peer -proc getPeer*(node: Eth2Node, peerId: PeerID): Peer {.gcsafe.} = - result = node.peers.getOrDefault(peerId) - if result == nil: - result = Peer.init(node, peerId) - node.peers[peerId] = result - -proc peerFromStream(daemon: DaemonAPI, stream: P2PStream): Peer {.gcsafe.} = - Eth2Node(daemon.userData).getPeer(stream.peer) +proc initializeConnection*(peer: Peer) {.async.} = + let daemon = peer.network.daemon + try: + peer.rpcStream = await daemon.openStream(peer.id, @[beaconChainProtocol]) + await performProtocolHandshakes(peer) + except CatchableError: + await reraiseAsPeerDisconnected(peer, "Failed to perform handshake") proc handleConnectingBeaconChainPeer(daemon: DaemonAPI, stream: P2PStream) {.async, gcsafe.} = let peer = daemon.peerFromStream(stream) + peer.rpcStream = stream + await performProtocolHandshakes(peer) proc accepts(d: Dispatcher, methodId: uint16): bool = methodId.int < d.messages.len diff --git a/tests/test_peer_connection.nim b/tests/test_peer_connection.nim new file mode 100644 index 0000000000..d5d923c29e --- /dev/null +++ b/tests/test_peer_connection.nim @@ -0,0 +1,24 @@ +import + unittest, os, + chronos, confutils, + ../beacon_chain/[conf, eth2_network] + +template asyncTest*(name, body: untyped) = + test name: + proc scenario {.async.} = body + waitFor scenario() + +asyncTest "connect two nodes": + let tempDir = getTempDir() / "peers_test" + + var c1 = BeaconNodeConf.defaults + c1.dataDir = OutDir(tempDir / "node-1") + var n1 = await createEth2Node(c1) + var n1Address = getPersistenBootstrapAddr(c1, parseIpAddress("127.0.0.1"), Port 50000) + + var c2 = BeaconNodeConf.defaults + c2.dataDir = OutDir(tempDir / "node-2") + var n2 = await createEth2Node(c2) + + await n2.connectToNetwork(bootstrapNodes = @[n1Address]) + From 15fdf78d6a5504afc3d190a9a1f8c2436a0c8c71 Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Wed, 12 Jun 2019 15:23:05 +0300 Subject: [PATCH 09/18] [LibP2P] Persistent network key for the bootstrap node --- beacon_chain/eth2_network.nim | 23 ++++++++++++++++------- beacon_chain/libp2p_backend.nim | 15 ++++++++------- beacon_chain/libp2p_spec_backend.nim | 15 ++++++++------- tests/test_peer_connection.nim | 16 ++++++++++++++-- 4 files changed, 46 insertions(+), 23 deletions(-) diff --git a/beacon_chain/eth2_network.nim b/beacon_chain/eth2_network.nim index 7f660b4352..80da0c98bf 100644 --- a/beacon_chain/eth2_network.nim +++ b/beacon_chain/eth2_network.nim @@ -118,8 +118,8 @@ when networkBackend == rlpxBackend: else: import - random, - libp2p/daemon/daemonapi, eth/async_utils, + os, random, std_shims/io, + libp2p/crypto/crypto, libp2p/daemon/daemonapi, eth/async_utils, ssz when networkBackend == libp2pSpecBackend: @@ -136,6 +136,9 @@ else: BootstrapAddr* = PeerInfo Eth2NodeIdentity* = PeerInfo + const + networkKeyFilename = "privkey.protobuf" + proc writeValue*(writer: var JsonWriter, value: PeerID) {.inline.} = writer.writeValue value.pretty @@ -151,20 +154,26 @@ else: proc init*(T: type BootstrapAddr, str: string): T = Json.decode(str, PeerInfo) - proc createEth2Node*(conf: BeaconNodeConf): Future[Eth2Node] {.async.} = - var node = new Eth2Node - await node.init() - return node + proc ensureNetworkIdFile(conf: BeaconNodeConf): string = + result = conf.dataDir / networkKeyFilename + if not fileExists(result): + createDir conf.dataDir.string + let pk = PrivateKey.random(Ed25519) + writeFile(result, pk.getBytes) proc getPersistentNetIdentity*(conf: BeaconNodeConf): Eth2NodeIdentity = # Using waitFor here is reasonable, because this proc is needed only # prior to connecting to the network. The RLPx alternative reads from # file and it's much easier to use if it's not async. # TODO: revisit in the future when we have our own Lib2P2 implementation. - let daemon = waitFor newDaemonApi() + let daemon = waitFor newDaemonApi(id = conf.ensureNetworkIdFile) result = waitFor daemon.identity() waitFor daemon.close() + proc createEth2Node*(conf: BeaconNodeConf): Future[Eth2Node] {.async.} = + var daemon = await newDaemonApi({PSGossipSub}, id = conf.ensureNetworkIdFile) + return await Eth2Node.init(daemon) + proc getPersistenBootstrapAddr*(conf: BeaconNodeConf, ip: IpAddress, port: Port): BootstrapAddr = # TODO what about the ports? diff --git a/beacon_chain/libp2p_backend.nim b/beacon_chain/libp2p_backend.nim index ac2bf3efc6..254b0258c6 100644 --- a/beacon_chain/libp2p_backend.nim +++ b/beacon_chain/libp2p_backend.nim @@ -91,19 +91,20 @@ include libp2p_backends_common include eth/p2p/p2p_backends_helpers include eth/p2p/p2p_tracing -proc init*(node: Eth2Node) {.async.} = - node.daemon = await newDaemonApi({PSGossipSub}) - node.daemon.userData = node - init node.peers +proc init*(T: type Eth2Node, daemon: DaemonAPI): Future[T] {.async.} = + new result + result.daemon = daemon + result.daemon.userData = result + init result.peers - newSeq node.protocolStates, allProtocols.len + newSeq result.protocolStates, allProtocols.len for proto in allProtocols: if proto.networkStateInitializer != nil: - node.protocolStates[proto.index] = proto.networkStateInitializer(node) + result.protocolStates[proto.index] = proto.networkStateInitializer(result) for msg in proto.messages: if msg.libp2pProtocol.len > 0: - await node.daemon.addHandler(@[msg.libp2pProtocol], msg.thunk) + await daemon.addHandler(@[msg.libp2pProtocol], msg.thunk) proc readMsg(stream: P2PStream, MsgType: type, deadline: Future[void]): Future[Option[MsgType]] {.async.} = diff --git a/beacon_chain/libp2p_spec_backend.nim b/beacon_chain/libp2p_spec_backend.nim index eca716db79..cb17794e6c 100644 --- a/beacon_chain/libp2p_spec_backend.nim +++ b/beacon_chain/libp2p_spec_backend.nim @@ -160,17 +160,18 @@ include eth/p2p/p2p_tracing proc handleConnectingBeaconChainPeer(daemon: DaemonAPI, stream: P2PStream) {.async, gcsafe.} -proc init*(node: Eth2Node) {.async.} = - node.daemon = await newDaemonApi({PSGossipSub}) - node.daemon.userData = node - init node.peers +proc init*(T: type Eth2Node, daemon: DaemonAPI): Future[Eth2Node] {.async.} = + new result + result.daemon = daemon + result.daemon.userData = result + result.peers = initTable[PeerID, Peer]() - newSeq node.protocolStates, allProtocols.len + newSeq result.protocolStates, allProtocols.len for proto in allProtocols: if proto.networkStateInitializer != nil: - node.protocolStates[proto.index] = proto.networkStateInitializer(node) + result.protocolStates[proto.index] = proto.networkStateInitializer(result) - await node.daemon.addHandler(@[beaconChainProtocol], handleConnectingBeaconChainPeer) + await daemon.addHandler(@[beaconChainProtocol], handleConnectingBeaconChainPeer) proc init*(T: type Peer, network: Eth2Node, id: PeerID): Peer = new result diff --git a/tests/test_peer_connection.nim b/tests/test_peer_connection.nim index d5d923c29e..5de2b3d82e 100644 --- a/tests/test_peer_connection.nim +++ b/tests/test_peer_connection.nim @@ -13,12 +13,24 @@ asyncTest "connect two nodes": var c1 = BeaconNodeConf.defaults c1.dataDir = OutDir(tempDir / "node-1") + + var n1PersistentAddress = c1.getPersistenBootstrapAddr( + parseIpAddress("127.0.0.1"), Port 50000) + var n1 = await createEth2Node(c1) - var n1Address = getPersistenBootstrapAddr(c1, parseIpAddress("127.0.0.1"), Port 50000) + var n1ActualAddress = await n1.daemon.identity() + + echo "persistent: ", n1PersistentAddress + echo "actual:", n1ActualAddress + doAssert n1PersistentAddress == n1ActualAddress + + echo "Node 1 address: ", n1PersistentAddress + echo "Press any key to continue" + discard stdin.readLine() var c2 = BeaconNodeConf.defaults c2.dataDir = OutDir(tempDir / "node-2") var n2 = await createEth2Node(c2) - await n2.connectToNetwork(bootstrapNodes = @[n1Address]) + await n2.connectToNetwork(bootstrapNodes = @[n1ActualAddress]) From 26ef341760add5e8885b23f12a42df52a4eb6a15 Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Wed, 12 Jun 2019 18:57:05 +0300 Subject: [PATCH 10/18] Use the libp2p_spec back-end by default --- nim.cfg | 2 -- 1 file changed, 2 deletions(-) diff --git a/nim.cfg b/nim.cfg index 869167e4fe..049d31d895 100644 --- a/nim.cfg +++ b/nim.cfg @@ -1,8 +1,6 @@ --threads:on --opt:speed ---define:"network_type=rlpx" - @if windows: # increase stack size --passL:"-Wl,--stack,8388608" From 96e2a02faf637814d23c23dcf7f79b5936500cf5 Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Wed, 12 Jun 2019 21:22:05 +0300 Subject: [PATCH 11/18] Respect the port configuration and NAT setup when using LibP2P --- beacon_chain/eth2_network.nim | 101 +++++++++++++++++++-------------- tests/test_peer_connection.nim | 18 +++--- 2 files changed, 69 insertions(+), 50 deletions(-) diff --git a/beacon_chain/eth2_network.nim b/beacon_chain/eth2_network.nim index 80da0c98bf..110a0ee5c2 100644 --- a/beacon_chain/eth2_network.nim +++ b/beacon_chain/eth2_network.nim @@ -1,16 +1,58 @@ import options, tables, - chronos, json_serialization, strutils, - chronicles, + chronos, json_serialization, strutils, chronicles, eth/net/nat, spec/digest, version, conf const clientId = "Nimbus beacon node v" & fullVersionStr() +export + version + +let + globalListeningAddr = parseIpAddress("0.0.0.0") + +proc setupNat(conf: BeaconNodeConf): tuple[ip: IpAddress, + tcpPort: Port, + udpPort: Port] = + # defaults + result.ip = globalListeningAddr + result.tcpPort = Port(conf.tcpPort) + result.udpPort = Port(conf.udpPort) + + var nat: NatStrategy + case conf.nat.toLowerAscii: + of "any": + nat = NatAny + of "none": + nat = NatNone + of "upnp": + nat = NatUpnp + of "pmp": + nat = NatPmp + else: + if conf.nat.startsWith("extip:") and isIpAddress(conf.nat[6..^1]): + # any required port redirection is assumed to be done by hand + result.ip = parseIpAddress(conf.nat[6..^1]) + nat = NatNone + else: + error "not a valid NAT mechanism, nor a valid IP address", value = conf.nat + quit(QuitFailure) + + if nat != NatNone: + let extIP = getExternalIP(nat) + if extIP.isSome: + result.ip = extIP.get() + let extPorts = redirectPorts(tcpPort = result.tcpPort, + udpPort = result.udpPort, + description = clientId) + if extPorts.isSome: + (result.tcpPort, result.udpPort) = extPorts.get() + when networkBackend == rlpxBackend: import os, - eth/[rlp, p2p, keys, net/nat], gossipsub_protocol, + eth/[rlp, p2p, keys], gossipsub_protocol, eth/p2p/peer_pool # for log on connected peers export @@ -25,43 +67,6 @@ when networkBackend == rlpxBackend: Eth2NodeIdentity* = KeyPair BootstrapAddr* = ENode - template libp2pProtocol*(name, version: string) {.pragma.} - - proc setupNat(conf: BeaconNodeConf): tuple[ip: IpAddress, tcpPort: Port, udpPort: Port] = - # defaults - result.ip = parseIpAddress("127.0.0.1") - result.tcpPort = Port(conf.tcpPort) - result.udpPort = Port(conf.udpPort) - - var nat: NatStrategy - case conf.nat.toLowerAscii: - of "any": - nat = NatAny - of "none": - nat = NatNone - of "upnp": - nat = NatUpnp - of "pmp": - nat = NatPmp - else: - if conf.nat.startsWith("extip:") and isIpAddress(conf.nat[6..^1]): - # any required port redirection is assumed to be done by hand - result.ip = parseIpAddress(conf.nat[6..^1]) - nat = NatNone - else: - error "not a valid NAT mechanism, nor a valid IP address", value = conf.nat - quit(QuitFailure) - - if nat != NatNone: - let extIP = getExternalIP(nat) - if extIP.isSome: - result.ip = extIP.get() - let extPorts = redirectPorts(tcpPort = result.tcpPort, - udpPort = result.udpPort, - description = clientId) - if extPorts.isSome: - (result.tcpPort, result.udpPort) = extPorts.get() - proc getPersistentNetIdentity*(conf: BeaconNodeConf): Eth2NodeIdentity = let privateKeyFile = conf.dataDir / "network.privkey" var privKey: PrivateKey @@ -171,13 +176,23 @@ else: waitFor daemon.close() proc createEth2Node*(conf: BeaconNodeConf): Future[Eth2Node] {.async.} = - var daemon = await newDaemonApi({PSGossipSub}, id = conf.ensureNetworkIdFile) + var + (extIp, extTcpPort, extUdpPort) = setupNat(conf) + hostAddress = tcpEndPoint(globalListeningAddr, Port conf.tcpPort) + announcedAddresses = if extIp != globalListeningAddr: @[] + else: @[tcpEndPoint(extIp, extTcpPort)] + + daemon = await newDaemonApi({PSGossipSub}, + id = conf.ensureNetworkIdFile, + hostAddresses = @[hostAddress], + announcedAddresses = announcedAddresses) + return await Eth2Node.init(daemon) proc getPersistenBootstrapAddr*(conf: BeaconNodeConf, ip: IpAddress, port: Port): BootstrapAddr = - # TODO what about the ports? - getPersistentNetIdentity(conf) + result = getPersistentNetIdentity(conf) + result.addresses = @[tcpEndPoint(ip, port)] proc isSameNode*(bootstrapNode: BootstrapAddr, id: Eth2NodeIdentity): bool = bootstrapNode == id diff --git a/tests/test_peer_connection.nim b/tests/test_peer_connection.nim index 5de2b3d82e..86e10a88b4 100644 --- a/tests/test_peer_connection.nim +++ b/tests/test_peer_connection.nim @@ -13,24 +13,28 @@ asyncTest "connect two nodes": var c1 = BeaconNodeConf.defaults c1.dataDir = OutDir(tempDir / "node-1") + c1.tcpPort = 50000 + c1.nat = "none" var n1PersistentAddress = c1.getPersistenBootstrapAddr( - parseIpAddress("127.0.0.1"), Port 50000) + parseIpAddress("127.0.0.1"), Port c1.tcpPort) var n1 = await createEth2Node(c1) - var n1ActualAddress = await n1.daemon.identity() - echo "persistent: ", n1PersistentAddress - echo "actual:", n1ActualAddress - doAssert n1PersistentAddress == n1ActualAddress + echo "Node 1 persistent address: ", n1PersistentAddress + + when networkBackend != rlpxBackend: + var n1ActualAddress = await n1.daemon.identity() + echo "Node 1 actual address:", n1ActualAddress - echo "Node 1 address: ", n1PersistentAddress echo "Press any key to continue" discard stdin.readLine() var c2 = BeaconNodeConf.defaults c2.dataDir = OutDir(tempDir / "node-2") + c2.tcpPort = 50001 + c2.nat = "none" var n2 = await createEth2Node(c2) - await n2.connectToNetwork(bootstrapNodes = @[n1ActualAddress]) + await n2.connectToNetwork(bootstrapNodes = @[n1PersistentAddress]) From e228c2dbcbe40bd2b5fd1ca89043abe205775a49 Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Mon, 17 Jun 2019 14:08:05 +0300 Subject: [PATCH 12/18] Implement the even/odd request ID scheme; Handle more edge cases; Break the cyclic imports --- beacon_chain/beacon_node.nim | 19 +- beacon_chain/beacon_node_types.nim | 1 + beacon_chain/eth2_network.nim | 15 +- beacon_chain/libp2p_backend.nim | 4 +- beacon_chain/libp2p_spec_backend.nim | 400 +++++++++++++++------------ beacon_chain/sync_protocol.nim | 10 +- 6 files changed, 254 insertions(+), 195 deletions(-) diff --git a/beacon_chain/beacon_node.nim b/beacon_chain/beacon_node.nim index d4d564de52..a1f2b8ee34 100644 --- a/beacon_chain/beacon_node.nim +++ b/beacon_chain/beacon_node.nim @@ -4,10 +4,10 @@ import chronos, chronicles, confutils, serialization/errors, eth/trie/db, eth/trie/backends/rocksdb_backend, eth/async_utils, spec/[bitfield, datatypes, digest, crypto, beaconstate, helpers, validator], - conf, time, - state_transition, fork_choice, ssz, beacon_chain_db, validator_pool, extras, - attestation_pool, block_pool, eth2_network, beacon_node_types, - mainchain_monitor, trusted_state_snapshots, version + conf, time, state_transition, fork_choice, ssz, beacon_chain_db, + validator_pool, extras, attestation_pool, block_pool, eth2_network, + beacon_node_types, mainchain_monitor, trusted_state_snapshots, version, + sync_protocol, request_manager const topicBeaconBlocks = "ethereum/2.1/beacon_chain/blocks" @@ -18,13 +18,7 @@ const genesisFile = "genesis.json" testnetsBaseUrl = "https://serenity-testnets.status.im" -# ################################################# -# Careful handling of beacon_node <-> sync_protocol -# to avoid recursive dependencies proc onBeaconBlock*(node: BeaconNode, blck: BeaconBlock) {.gcsafe.} - # Forward decl for sync_protocol -import sync_protocol, request_manager -# ################################################# func localValidatorsDir(conf: BeaconNodeConf): string = conf.dataDir / "validators" @@ -88,6 +82,7 @@ proc saveValidatorKey(keyName, key: string, conf: BeaconNodeConf) = proc init*(T: type BeaconNode, conf: BeaconNodeConf): Future[BeaconNode] {.async.} = new result + result.onBeaconBlock = onBeaconBlock result.config = conf result.networkIdentity = getPersistentNetIdentity(conf) result.nickname = if conf.nodename == "auto": shortForm(result.networkIdentity) @@ -654,7 +649,7 @@ proc onSecond(node: BeaconNode, moment: Moment) {.async.} = if missingBlocks.len > 0: info "Requesting detected missing blocks", missingBlocks node.requestManager.fetchAncestorBlocks(missingBlocks) do (b: BeaconBlock): - node.onBeaconBlock(b) + onBeaconBlock(node ,b) let nextSecond = max(Moment.now(), moment + chronos.seconds(1)) addTimer(nextSecond) do (p: pointer): @@ -662,7 +657,7 @@ proc onSecond(node: BeaconNode, moment: Moment) {.async.} = proc run*(node: BeaconNode) = waitFor node.network.subscribe(topicBeaconBlocks) do (blck: BeaconBlock): - node.onBeaconBlock(blck) + onBeaconBlock(node, blck) waitFor node.network.subscribe(topicAttestations) do (attestation: Attestation): node.onAttestation(attestation) diff --git a/beacon_chain/beacon_node_types.nim b/beacon_chain/beacon_node_types.nim index 0e5306d3f0..a0a00f5348 100644 --- a/beacon_chain/beacon_node_types.nim +++ b/beacon_chain/beacon_node_types.nim @@ -25,6 +25,7 @@ type attestationPool*: AttestationPool mainchainMonitor*: MainchainMonitor beaconClock*: BeaconClock + onBeaconBlock*: proc (node: BeaconNode, blck: BeaconBlock) {.gcsafe.} stateCache*: StateData ##\ ## State cache object that's used as a scratch pad diff --git a/beacon_chain/eth2_network.nim b/beacon_chain/eth2_network.nim index 110a0ee5c2..5c9bf03b3e 100644 --- a/beacon_chain/eth2_network.nim +++ b/beacon_chain/eth2_network.nim @@ -175,17 +175,22 @@ else: result = waitFor daemon.identity() waitFor daemon.close() + template tcpEndPoint(address, port): auto = + MultiAddress.init(address, IPPROTO_TCP, port) + proc createEth2Node*(conf: BeaconNodeConf): Future[Eth2Node] {.async.} = var (extIp, extTcpPort, extUdpPort) = setupNat(conf) hostAddress = tcpEndPoint(globalListeningAddr, Port conf.tcpPort) announcedAddresses = if extIp != globalListeningAddr: @[] else: @[tcpEndPoint(extIp, extTcpPort)] + keyFile = conf.ensureNetworkIdFile - daemon = await newDaemonApi({PSGossipSub}, - id = conf.ensureNetworkIdFile, - hostAddresses = @[hostAddress], - announcedAddresses = announcedAddresses) + info "Starting LibP2P deamon", hostAddress, announcedAddresses, keyFile + let daemon = await newDaemonApi({PSGossipSub}, + id = keyFile, + hostAddresses = @[hostAddress], + announcedAddresses = announcedAddresses) return await Eth2Node.init(daemon) @@ -195,7 +200,7 @@ else: result.addresses = @[tcpEndPoint(ip, port)] proc isSameNode*(bootstrapNode: BootstrapAddr, id: Eth2NodeIdentity): bool = - bootstrapNode == id + bootstrapNode.peer == id.peer proc shortForm*(id: Eth2NodeIdentity): string = # TODO: Make this shorter diff --git a/beacon_chain/libp2p_backend.nim b/beacon_chain/libp2p_backend.nim index 254b0258c6..c3d949e7f0 100644 --- a/beacon_chain/libp2p_backend.nim +++ b/beacon_chain/libp2p_backend.nim @@ -22,6 +22,7 @@ type connectionState*: ConnectionState awaitedMessages: Table[CompressedMsgId, FutureBase] protocolStates*: seq[RootRef] + maxInactivityAllowed*: Duration ConnectionState* = enum None, @@ -95,6 +96,7 @@ proc init*(T: type Eth2Node, daemon: DaemonAPI): Future[T] {.async.} = new result result.daemon = daemon result.daemon.userData = result + result.maxInactivityAllowed = 15.minutes # TODO: Read this from the config init result.peers newSeq result.protocolStates, allProtocols.len @@ -319,7 +321,7 @@ proc implementSendProcBody(sendProc: SendProc) = else: quote: `sendBytes`(`UntypedResponder`(`peer`).stream, `bytes`) - sendProc.useStandardBody(nil, sendCallGenerator) + sendProc.useStandardBody(nil, nil, sendCallGenerator) proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = var diff --git a/beacon_chain/libp2p_spec_backend.nim b/beacon_chain/libp2p_spec_backend.nim index cb17794e6c..3eb4e44a6c 100644 --- a/beacon_chain/libp2p_spec_backend.nim +++ b/beacon_chain/libp2p_spec_backend.nim @@ -1,7 +1,7 @@ import tables, deques, options, algorithm, std_shims/[macros_shim, tables_shims], ranges/ptr_arith, chronos, chronicles, serialization, faststreams/input_stream, - eth/p2p/p2p_protocol_dsl, libp2p/daemon/daemonapi, + eth/async_utils, eth/p2p/p2p_protocol_dsl, libp2p/daemon/daemonapi, ssz export @@ -9,10 +9,10 @@ export const # Compression nibble - NoCompression* = uint 0 + NoCompression* = byte 0 # Encoding nibble - SszEncoding* = uint 1 + SszEncoding* = byte 1 beaconChainProtocol = "/eth/serenity/beacon/rpc/1" @@ -27,13 +27,13 @@ type Peer* = ref object network*: Eth2Node id*: PeerID - lastSentMsgId*: uint64 + lastReqId*: uint64 rpcStream*: P2PStream connectionState*: ConnectionState awaitedMessages: Table[CompressedMsgId, FutureBase] - outstandingRequests*: seq[Deque[OutstandingRequest]] + outstandingRequests*: Table[uint64, OutstandingRequest] protocolStates*: seq[RootRef] - maxInactivityAllowed: Duration + maxInactivityAllowed*: Duration ConnectionState* = enum None, @@ -69,6 +69,7 @@ type id*: uint64 future*: FutureBase timeoutAt*: Moment + responseThunk*: ThunkProc ProtocolConnection* = object stream*: P2PStream @@ -96,13 +97,9 @@ type networkStateInitializer*: NetworkStateInitializer handshake*: HandshakeStep disconnectHandler*: DisconnectionHandler - dispatcher: Dispatcher ProtocolInfo* = ptr ProtocolInfoObj - Dispatcher* = object - messages*: seq[MessageInfo] - SpecOuterMsgHeader {.packed.} = object compression {.bitsize: 4.}: uint encoding {.bitsize: 4.}: uint @@ -112,6 +109,10 @@ type reqId: uint64 methodId: uint16 + ErrorResponse {.packed.} = object + outerHeader: SpecOuterMsgHeader + innerHeader: SpecInnerMsgHeader + PeerStateInitializer* = proc(peer: Peer): RootRef {.gcsafe.} NetworkStateInitializer* = proc(network: Eth2Node): RootRef {.gcsafe.} @@ -121,6 +122,7 @@ type ThunkProc* = proc(peer: Peer, stream: P2PStream, reqId: uint64, + reqFuture: FutureBase, msgData: ByteStreamVar): Future[void] {.gcsafe.} MessageContentPrinter* = proc(msg: pointer): string {.gcsafe.} @@ -147,6 +149,14 @@ const BreachOfProtocol* = FaultOrError # TODO: We should lobby for more disconnection reasons. +template isOdd(val: SomeInteger): bool = + type T = type(val) + (val and T(1)) != 0 + +proc init(T: type SpecOuterMsgHeader, + compression, encoding: byte, msgLen: uint64): T = + T(compression: compression, encoding: encoding, msgLen: msgLen) + proc readPackedObject(stream: P2PStream, T: type): Future[T] {.async.} = await stream.transp.readExactly(addr result, sizeof result) @@ -154,6 +164,10 @@ proc appendPackedObject(stream: OutputStreamVar, value: auto) = let valueAsBytes = cast[ptr byte](unsafeAddr(value)) stream.append makeOpenArray(valueAsBytes, sizeof(value)) +proc getThunk(protocol: ProtocolInfo, methodId: uint16): ThunkProc = + if methodId.int >= protocol.messages.len: return nil + protocol.messages[methodId.int].thunk + include libp2p_backends_common include eth/p2p/p2p_backends_helpers include eth/p2p/p2p_tracing @@ -178,7 +192,8 @@ proc init*(T: type Peer, network: Eth2Node, id: PeerID): Peer = result.id = id result.network = network result.awaitedMessages = initTable[CompressedMsgId, FutureBase]() - result.connectionState = Connected + result.maxInactivityAllowed = 15.minutes # TODO: read this from the config + result.connectionState = None newSeq result.protocolStates, allProtocols.len for i in 0 ..< allProtocols.len: let proto = allProtocols[i] @@ -189,52 +204,51 @@ proc init*[MsgName](T: type ResponderWithId[MsgName], peer: Peer, reqId: uint64): T = T(peer: peer, reqId: reqId) -proc performProtocolHandshakes*(peer: Peer) {.async.} = - var subProtocolsHandshakes = newSeqOfCap[Future[void]](allProtocols.len) - for protocol in allProtocols: - if protocol.handshake != nil: - subProtocolsHandshakes.add((protocol.handshake)(peer, peer.rpcStream)) - - await all(subProtocolsHandshakes) - debug "All protocols initialized", peer - -proc initializeConnection*(peer: Peer) {.async.} = - let daemon = peer.network.daemon +proc sendMsg*(peer: Peer, data: Bytes) {.gcsafe, async.} = try: - peer.rpcStream = await daemon.openStream(peer.id, @[beaconChainProtocol]) - await performProtocolHandshakes(peer) + var unsentBytes = data.len + while true: + # TODO: this looks wrong. + # We are always trying to write the same data. + # Find all other places where such code is used. + unsentBytes -= await peer.rpcStream.transp.write(data) + if unsentBytes <= 0: return except CatchableError: - await reraiseAsPeerDisconnected(peer, "Failed to perform handshake") - -proc handleConnectingBeaconChainPeer(daemon: DaemonAPI, stream: P2PStream) {.async, gcsafe.} = - let peer = daemon.peerFromStream(stream) - peer.rpcStream = stream - await performProtocolHandshakes(peer) - -proc accepts(d: Dispatcher, methodId: uint16): bool = - methodId.int < d.messages.len - -proc invokeThunk(peer: Peer, - protocol: ProtocolInfo, - stream: P2PStream, - methodId: int, - reqId: uint64, - msgContents: ByteStreamVar): Future[void] = - template raiseInvalidMsgId = - raise newException(InvalidMsgIdError, - "ETH2 message with an invalid id " & $methodId) - - if methodId >= protocol.dispatcher.messages.len: raiseInvalidMsgId() - var thunk = protocol.dispatcher.messages[methodId].thunk - if thunk == nil: raiseInvalidMsgId() + await peer.disconnect(FaultOrError) + # this is usually a "(32) Broken pipe": + # FIXME: this exception should be caught somewhere in addMsgHandler() and + # sending should be retried a few times + raise - return thunk(peer, stream, reqId, msgContents) +proc sendMsg*[T](responder: ResponderWithId[T], data: Bytes): Future[void] = + return sendMsg(responder.peer, data) -proc recvAndDispatchMsg*(peer: Peer, protocol: ProtocolInfo, stream: P2PStream): - Future[PeerLoopExitReason] {.async.} = +proc sendErrorResponse(peer: Peer, reqId: uint64, + responseCode: ResponseCode): Future[void] = + var resp = ErrorResponse( + outerHeader: SpecOuterMsgHeader.init( + compression = NoCompression, + encoding = SszEncoding, + msgLen = uint64 sizeof(SpecInnerMsgHeader)), + innerHeader: SpecInnerMsgHeader( + reqId: reqId, + methodId: uint16(responseCode))) + + # TODO: don't allocate the Bytes sequence here + return peer.sendMsg @(makeOpenArray(cast[ptr byte](addr resp), sizeof resp)) + +proc recvAndDispatchMsg*(peer: Peer): Future[PeerLoopExitReason] {.async.} = template fail(reason) = return reason + # For now, we won't try to handle the presence of multiple sub-protocols + # since the spec is not defining how they will be mapped to P2P streams. + doAssert allProtocols.len == 1 + + var + stream = peer.rpcStream + protocol = allProtocols[0] + var outerHeader = await stream.readPackedObject(SpecOuterMsgHeader) if outerHeader.compression != NoCompression: @@ -246,44 +260,51 @@ proc recvAndDispatchMsg*(peer: Peer, protocol: ProtocolInfo, stream: P2PStream): if outerHeader.msgLen <= SpecInnerMsgHeader.sizeof.uint64: fail ProtocolViolation - var innerHeader = await stream.readPackedObject(SpecInnerMsgHeader) + let + innerHeader = await stream.readPackedObject(SpecInnerMsgHeader) + reqId = innerHeader.reqId var msgContent = newSeq[byte](outerHeader.msgLen - SpecInnerMsgHeader.sizeof.uint64) await stream.transp.readExactly(addr msgContent[0], msgContent.len) var msgContentStream = memoryStream(msgContent) - if protocol.dispatcher.accepts(innerHeader.methodId): - try: - await invokeThunk(peer, protocol, stream, - innerHeader.methodId.int, - innerHeader.reqId, - msgContentStream) - except SerializationError: - fail ProtocolViolation - except CatchableError: - warn "" - -proc sendMsg*(peer: Peer, data: Bytes) {.gcsafe, async.} = - try: - var unsentBytes = data.len - while true: - unsentBytes -= await peer.rpcStream.transp.write(data) - if unsentBytes <= 0: return - except: - await peer.disconnect(FaultOrError) - # this is usually a "(32) Broken pipe": - # FIXME: this exception should be caught somewhere in addMsgHandler() and - # sending should be retried a few times - raise - -proc sendMsg*[T](responder: ResponderWithId[T], data: Bytes): Future[void] = - return sendMsg(responder.peer, data) + if reqId.isOdd: + peer.outstandingRequests.withValue(reqId, req): + let thunk = req.responseThunk + let reqFuture = req.future + peer.outstandingRequests.del(reqId) + + try: + await thunk(peer, stream, reqId, reqFuture, msgContentStream) + except SerializationError: + debug "Error during deserialization", err = getCurrentExceptionMsg() + fail ProtocolViolation + except CatchableError: + # TODO + warn "" + do: + debug "Ignoring late or invalid response ID", peer, id = reqId + # TODO: skip the message + else: + let thunk = protocol.getThunk(innerHeader.methodId) + if thunk != nil: + try: + await thunk(peer, stream, reqId, nil, msgContentStream) + except SerializationError: + debug "Error during deserialization", err = getCurrentExceptionMsg() + fail ProtocolViolation + except CatchableError: + # TODO + warn "" + else: + debug "P2P request method not found", methodId = innerHeader.methodId + await peer.sendErrorResponse(reqId, MethodNotFound) -proc dispatchMessages*(peer: Peer, protocol: ProtocolInfo, stream: P2PStream): - Future[PeerLoopExitReason] {.async.} = +proc dispatchMessages*(peer: Peer): Future[PeerLoopExitReason] {.async.} = while true: - let dispatchedMsgFut = recvAndDispatchMsg(peer, protocol, stream) + let dispatchedMsgFut = recvAndDispatchMsg(peer) + doAssert peer.maxInactivityAllowed.milliseconds > 0 yield dispatchedMsgFut or sleepAsync(peer.maxInactivityAllowed) if not dispatchedMsgFut.finished: return InactivePeer @@ -295,87 +316,70 @@ proc dispatchMessages*(peer: Peer, protocol: ProtocolInfo, stream: P2PStream): if status == Success: continue return status -proc registerRequest(peer: Peer, - protocol: ProtocolInfo, - timeout: Duration, - responseFuture: FutureBase, - responseMethodId: uint16): uint64 = - inc peer.lastSentMsgId - result = peer.lastSentMsgId +proc performProtocolHandshakes*(peer: Peer) {.async.} = + peer.initProtocolStates allProtocols + + # Please note that the ordering of operations here is important! + # + # We must first start all handshake procedures and give them a + # chance to send any initial packages they might require over + # the network and to yield on their `nextMsg` waits. + # + var subProtocolsHandshakes = newSeqOfCap[Future[void]](allProtocols.len) + for protocol in allProtocols: + if protocol.handshake != nil: + subProtocolsHandshakes.add((protocol.handshake)(peer, peer.rpcStream)) - let timeoutAt = Moment.fromNow(timeout) - let req = OutstandingRequest(id: result, - future: responseFuture, - timeoutAt: timeoutAt) - peer.outstandingRequests[responseMethodId.int].addLast req + # The `dispatchMesssages` loop must be started after this. + # Otherwise, we risk that some of the handshake packets sent by + # the other peer may arrrive too early and be processed before + # the handshake code got a change to wait for them. + # + var messageProcessingLoop = peer.dispatchMessages() + messageProcessingLoop.callback = proc(p: pointer) {.gcsafe.} = + if messageProcessingLoop.failed: + debug "Ending dispatchMessages loop", peer, + err = messageProcessingLoop.error.msg + else: + debug "Ending dispatchMessages", peer, + exitCode = messageProcessingLoop.read + traceAsyncErrors peer.disconnect(ClientShutdown) - let requestResolver = protocol.dispatcher.messages[responseMethodId.int].requestResolver - proc timeoutExpired(udata: pointer) = requestResolver(nil, responseFuture) + # The handshake may involve multiple async steps, so we wait + # here for all of them to finish. + # + await all(subProtocolsHandshakes) - addTimer(timeoutAt, timeoutExpired, nil) + peer.connectionState = Connected + debug "Peer connection initialized", peer -proc resolvePendingFutures(peer: Peer, protocol: ProtocolInfo, - methodId: int, msg: pointer, reqId: uint64) = - logScope: - msg = protocol.dispatcher.messages[methodId].name - msgContents = protocol.dispatcher.messages[methodId].printer(msg) - receivedReqId = reqId - remotePeer = peer.id +proc initializeConnection*(peer: Peer) {.async.} = + let daemon = peer.network.daemon + try: + peer.connectionState = Connecting + peer.rpcStream = await daemon.openStream(peer.id, @[beaconChainProtocol]) + await performProtocolHandshakes(peer) + except CatchableError: + await reraiseAsPeerDisconnected(peer, "Failed to perform handshake") - template resolve(future) = - (protocol.dispatcher.messages[methodId].requestResolver)(msg, future) +proc handleConnectingBeaconChainPeer(daemon: DaemonAPI, stream: P2PStream) {.async, gcsafe.} = + let peer = daemon.peerFromStream(stream) + peer.rpcStream = stream + peer.connectionState = Connecting + await performProtocolHandshakes(peer) - template outstandingReqs: auto = - peer.outstandingRequests[methodId] +proc resolvePendingFutures(peer: Peer, protocol: ProtocolInfo, + methodId: int, msg: pointer, reqFuture: FutureBase) = let msgId = (protocolIdx: protocol.index, methodId: methodId) + if peer.awaitedMessages[msgId] != nil: - let msgInfo = protocol.dispatcher.messages[methodId] + let msgInfo = protocol.messages[methodId] msgInfo.nextMsgResolver(msg, peer.awaitedMessages[msgId]) peer.awaitedMessages[msgId] = nil - # TODO: This is not completely sound because we are still using a global - # `reqId` sequence (the problem is that we might get a response ID that - # matches a request ID for a different type of request). To make the code - # correct, we can use a separate sequence per response type, but we have - # to first verify that the other Ethereum clients are supporting this - # correctly (because then, we'll be reusing the same reqIds for different - # types of requests). Alternatively, we can assign a separate interval in - # the `reqId` space for each type of response. - if reqId > peer.lastSentMsgId: - warn "RLPx response without a matching request" - return - - var idx = 0 - while idx < outstandingReqs.len: - template req: auto = outstandingReqs()[idx] - - if req.future.finished: - doAssert req.timeoutAt <= Moment.now() - # Here we'll remove the expired request by swapping - # it with the last one in the deque (if necessary): - if idx != outstandingReqs.len - 1: - req = outstandingReqs.popLast - continue - else: - outstandingReqs.shrink(fromLast = 1) - # This was the last item, so we don't have any - # more work to do: - return - - if req.id == reqId: - resolve req.future - # Here we'll remove the found request by swapping - # it with the last one in the deque (if necessary): - if idx != outstandingReqs.len - 1: - req = outstandingReqs.popLast - else: - outstandingReqs.shrink(fromLast = 1) - return - - inc idx - - debug "late or duplicate reply for a network request" + if reqFuture != nil and not reqFuture.finished: + protocol.messages[methodId].requestResolver(msg, reqFuture) proc initProtocol(name: string, version: int, peerInit: PeerStateInitializer, @@ -423,28 +427,72 @@ proc prepareRequest(peer: Peer, stream: OutputStreamVar, timeout: Duration, responseFuture: FutureBase): DelayedWriteCursor = + assert peer != nil and + protocol != nil and + responseFuture != nil and + responseMethodId.int < protocol.messages.len - let reqId = registerRequest(peer, protocol, timeout, - responseFuture, responseMethodId) + doAssert timeout.milliseconds > 0 result = stream.delayFixedSizeWrite sizeof(SpecOuterMsgHeader) + inc peer.lastReqId, 2 + let reqId = peer.lastReqId + stream.appendPackedObject SpecInnerMsgHeader( - reqId: reqId, - methodId: requestMethodId) + reqId: reqId, methodId: requestMethodId) + + template responseMsgInfo: auto = + protocol.messages[responseMethodId.int] + + let + requestResolver = responseMsgInfo.requestResolver + timeoutAt = Moment.fromNow(timeout) + + peer.outstandingRequests[reqId + 1] = OutstandingRequest( + id: reqId, + future: responseFuture, + timeoutAt: timeoutAt, + responseThunk: responseMsgInfo.thunk) + + proc timeoutExpired(udata: pointer) = + requestResolver(nil, responseFuture) + peer.outstandingRequests.del(reqId + 1) + + addTimer(timeoutAt, timeoutExpired, nil) proc prepareResponse(responder: ResponderWithId, stream: OutputStreamVar): DelayedWriteCursor = result = stream.delayFixedSizeWrite sizeof(SpecOuterMsgHeader) + stream.appendPackedObject SpecInnerMsgHeader( + reqId: responder.reqId + 1, + methodId: uint16(Success)) + +proc prepareMsg(peer: Peer, methodId: uint16, + stream: OutputStreamVar): DelayedWriteCursor = + result = stream.delayFixedSizeWrite sizeof(SpecOuterMsgHeader) + + inc peer.lastReqId, 2 + stream.appendPackedObject SpecInnerMsgHeader( + reqId: peer.lastReqId, methodId: methodId) + +proc finishOuterHeader(headerCursor: DelayedWriteCursor) = + var outerHeader = SpecOuterMsgHeader.init( + compression = NoCompression, + encoding = SszEncoding, + msgLen = uint64(headerCursor.totalBytesWrittenAfterCursor)) + + headerCursor.endWrite makeOpenArray(cast[ptr byte](addr outerHeader), + sizeof outerHeader) + proc implementSendProcBody(sendProc: SendProc) = let msg = sendProc.msg delayedWriteCursor = ident "delayedWriteCursor" peer = sendProc.peerParam - proc preludeGenerator(stream: NimNode): NimNode = - result = newStmtList() + proc preSerializationStep(stream: NimNode): NimNode = case msg.kind of msgRequest: let @@ -453,15 +501,22 @@ proc implementSendProcBody(sendProc: SendProc) = protocol = sendProc.msg.protocol.protocolInfoVar timeout = sendProc.timeoutParam - result.add quote do: - let `delayedWriteCursor` = `prepareRequest`( + quote do: + var `delayedWriteCursor` = prepareRequest( `peer`, `protocol`, `requestMethodId`, `responseMethodId`, `stream`, `timeout`, `resultIdent`) + of msgResponse: - result.add quote do: - let `delayedWriteCursor` = `prepareResponse`(`peer`, `stream`) - else: - discard + quote do: + var `delayedWriteCursor` = prepareResponse(`peer`, `stream`) + + of msgHandshake, msgNotification: + let methodId = newLit(msg.id) + quote do: + var `delayedWriteCursor` = prepareMsg(`peer`, `methodId`, `stream`) + + proc postSerializationStep(stream: NimNode): NimNode = + newCall(bindSym "finishOuterHeader", delayedWriteCursor) proc sendCallGenerator(peer, bytes: NimNode): NimNode = let @@ -471,7 +526,7 @@ proc implementSendProcBody(sendProc: SendProc) = if msg.kind == msgRequest: # In RLPx requests, the returned future was allocated here and passed - # to `registerRequest`. It's already assigned to the result variable + # to `prepareRequest`. It's already assigned to the result variable # of the proc, so we just wait for the sending operation to complete # and we return in a normal way. (the waiting is done, so we can catch # any possible errors). @@ -481,7 +536,10 @@ proc implementSendProcBody(sendProc: SendProc) = # `sendMsg` call. quote: return `sendCall` - sendProc.useStandardBody(preludeGenerator, sendCallGenerator) + sendProc.useStandardBody( + preSerializationStep, + postSerializationStep, + sendCallGenerator) proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = let @@ -492,7 +550,7 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = Format = ident "SSZ" Response = bindSym "Response" ResponderWithId = bindSym "ResponderWithId" - perProtocolMsgId = ident"perProtocolMsgId" + perProtocolMsgId = ident "perProtocolMsgId" mount = bindSym "mount" @@ -508,6 +566,7 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = stream = ident "stream" protocol = ident "protocol" response = ident "response" + reqFutureVar = ident "reqFuture" msgContents = ident "msgContents" receivedMsg = ident "receivedMsg" @@ -546,12 +605,12 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = else: newStmtList() - let callResolvePendingFutures = if msg.kind == msgResponse: - newCall(resolvePendingFutures, - peerVar, protocol.protocolInfoVar, - msgIdLit, newCall("addr", receivedMsg), reqIdVar) - else: - newStmtList() + let callResolvePendingFutures = newCall( + resolvePendingFutures, peerVar, + protocol.protocolInfoVar, + msgIdLit, + newCall("addr", receivedMsg), + reqFutureVar) var userHandlerParams = @[peerVar] if msg.kind == msgRequest: @@ -565,6 +624,7 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = proc `thunkName`(`peerVar`: `Peer`, `stream`: `P2PStream`, `reqIdVar`: uint64, + `reqFutureVar`: FutureBase, `msgContents`: `ByteStreamVar`) {.async, gcsafe.} = var `receivedMsg` = `mount`(`Format`, `msgContents`, `msgRecName`) `traceMsg` diff --git a/beacon_chain/sync_protocol.nim b/beacon_chain/sync_protocol.nim index 9ab26f2fb0..ecd777790c 100644 --- a/beacon_chain/sync_protocol.nim +++ b/beacon_chain/sync_protocol.nim @@ -4,10 +4,6 @@ import spec/[datatypes, crypto, digest, helpers], eth/rlp, beacon_node_types, eth2_network, beacon_chain_db, block_pool, time, ssz -from beacon_node import onBeaconBlock - # Careful handling of beacon_node <-> sync_protocol - # to avoid recursive dependencies - type ValidatorChangeLogEntry* = object case kind*: ValidatorSetDeltaFlags @@ -49,7 +45,7 @@ proc fromHeaderAndBody(b: var BeaconBlock, h: BeaconBlockHeader, body: BeaconBlo proc importBlocks(node: BeaconNode, blocks: openarray[BeaconBlock]) = for blk in blocks: - node.onBeaconBlock(blk) + node.onBeaconBlock(node, blk) info "Forward sync imported blocks", len = blocks.len proc mergeBlockHeadersAndBodies(headers: openarray[BeaconBlockHeader], bodies: openarray[BeaconBlockBody]): Option[seq[BeaconBlock]] = @@ -105,7 +101,7 @@ p2pProtocol BeaconSync(version = 1, let bestDiff = cmp((latestFinalizedEpoch, bestSlot), (m.latestFinalizedEpoch, m.bestSlot)) if bestDiff >= 0: # Nothing to do? - trace "Nothing to sync", peer = peer.remote + debug "Nothing to sync", peer else: # TODO: Check for WEAK_SUBJECTIVITY_PERIOD difference and terminate the # connection if it's too big. @@ -124,7 +120,7 @@ p2pProtocol BeaconSync(version = 1, if lastSlot <= s: info "Slot did not advance during sync", peer break - + s = lastSlot + 1 else: break From 25494aa2fb3b2d6c92518d6587192417e213e19b Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Wed, 19 Jun 2019 16:41:54 +0300 Subject: [PATCH 13/18] Better error message on incompatible validator data when resetting a testnet --- beacon_chain/beacon_node.nim | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/beacon_chain/beacon_node.nim b/beacon_chain/beacon_node.nim index a1f2b8ee34..13163b511f 100644 --- a/beacon_chain/beacon_node.nim +++ b/beacon_chain/beacon_node.nim @@ -722,7 +722,13 @@ when isMainModule: for i in config.firstValidator.int ..< config.totalValidators.int: let depositFile = config.validatorsDir / validatorFileBaseName(i) & ".deposit.json" - deposits.add Json.loadFile(depositFile, Deposit) + try: + deposits.add Json.loadFile(depositFile, Deposit) + except SerializationError as err: + stderr.write "Error while loading a deposit file:\n" + stderr.write err.formatMsg(depositFile), "\n" + stderr.write "Please regenerate the deposit files by running validator_keygen again\n" + quit 1 let initialState = get_genesis_beacon_state( deposits, From 976438d7fc56efcede46ede85f530d81e29055be Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Wed, 19 Jun 2019 21:12:09 +0300 Subject: [PATCH 14/18] reset all testnet flavours together --- scripts/reset_testnet.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/reset_testnet.sh b/scripts/reset_testnet.sh index 1b61789f3d..827c6a2ba7 100755 --- a/scripts/reset_testnet.sh +++ b/scripts/reset_testnet.sh @@ -28,7 +28,7 @@ regenTestnetFiles() { --outputDir="$NETWORK_DIR" fi - nim c -r $NIM_FLAGS beacon_chain/beacon_node \ + nim c -d:"network_type=$NETWORK_FLAVOUR" -r $NIM_FLAGS beacon_chain/beacon_node \ --network=$NETWORK_NAME \ --dataDir=$DATA_DIR/node-0 \ createTestnet \ @@ -44,5 +44,6 @@ regenTestnetFiles() { } regenTestnetFiles rlpx -# regenTestnetFiles libp2p -d:withLibP2P +regenTestnetFiles libp2p_spec +regenTestnetFiles libp2p_native From 44fc05f864b5a59adebf1d65f054deaee45ae29b Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Thu, 20 Jun 2019 15:45:09 +0300 Subject: [PATCH 15/18] Fix the build for chronicles_sinks=json --- beacon_chain/libp2p_backends_common.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/beacon_chain/libp2p_backends_common.nim b/beacon_chain/libp2p_backends_common.nim index 24318727c8..e2d7353767 100644 --- a/beacon_chain/libp2p_backends_common.nim +++ b/beacon_chain/libp2p_backends_common.nim @@ -1,6 +1,8 @@ # included from libp2p_backend -proc `$`*(peer: Peer): string = $peer.id +template `$`*(peer: Peer): string = $peer.id + +chronicles.formatIt(Peer): $it proc init*(T: type Peer, network: Eth2Node, id: PeerID): Peer {.gcsafe.} From 1bcd94a7dafb35210d33e75d8ff140e69cd198eb Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Thu, 20 Jun 2019 16:43:53 +0300 Subject: [PATCH 16/18] Switch to a model where the testnets are stricly associated with a network back-end type --- scripts/build_testnet_node.sh | 4 ++-- scripts/reset_testnet.sh | 6 +++--- scripts/testnet0.env | 1 + scripts/testnet1.env | 1 + 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/scripts/build_testnet_node.sh b/scripts/build_testnet_node.sh index 81c8898b15..a3b2a84c2a 100755 --- a/scripts/build_testnet_node.sh +++ b/scripts/build_testnet_node.sh @@ -12,7 +12,7 @@ source "$NETWORK_NAME.env" cd .. -NIM_FLAGS="-d:release --lineTrace:on -d:chronicles_log_level=DEBUG -d:SECONDS_PER_SLOT=$SECONDS_PER_SLOT -d:SHARD_COUNT=$SHARD_COUNT -d:SLOTS_PER_EPOCH=$SLOTS_PER_EPOCH -d:DEFAULT_NETWORK=$NETWORK_NAME --hints:off --verbosity:0" +NIM_FLAGS="-d:release --lineTrace:on -d:chronicles_log_level=DEBUG -d:network_type=$NETWORK_TYPE -d:SECONDS_PER_SLOT=$SECONDS_PER_SLOT -d:SHARD_COUNT=$SHARD_COUNT -d:SLOTS_PER_EPOCH=$SLOTS_PER_EPOCH -d:DEFAULT_NETWORK=$NETWORK_NAME --hints:off --verbosity:0" BEACON_NODE_BIN="build/${NETWORK_NAME}_node" @@ -29,7 +29,7 @@ echo "Done! You're now ready to connect to $NETWORK_NAME by running:" echo echo " $BEACON_NODE_BIN" echo -echo "Database and configuration files placed in:" +echo "Database and configuration files will be placed in:" echo echo " ${HOME}/.cache/nimbus/BeaconNode/${NETWORK_NAME}" echo diff --git a/scripts/reset_testnet.sh b/scripts/reset_testnet.sh index 827c6a2ba7..781ab85deb 100755 --- a/scripts/reset_testnet.sh +++ b/scripts/reset_testnet.sh @@ -19,7 +19,7 @@ NETWORK_DIR=$WWW_DIR/$NETWORK_NAME regenTestnetFiles() { NIM_FLAGS="-d:release -d:SECONDS_PER_SLOT=$SECONDS_PER_SLOT -d:SHARD_COUNT=$SHARD_COUNT -d:SLOTS_PER_EPOCH=$SLOTS_PER_EPOCH ${2:-}" - NETWORK_FLAVOUR=$1 + NETWORK_TYPE=$1 if [ ! -f $NETWORK_DIR/genesis.json ]; then rm -f $NETWORK_DIR/* @@ -28,7 +28,7 @@ regenTestnetFiles() { --outputDir="$NETWORK_DIR" fi - nim c -d:"network_type=$NETWORK_FLAVOUR" -r $NIM_FLAGS beacon_chain/beacon_node \ + nim c -d:"network_type=$NETWORK_TYPE" -r $NIM_FLAGS beacon_chain/beacon_node \ --network=$NETWORK_NAME \ --dataDir=$DATA_DIR/node-0 \ createTestnet \ @@ -37,7 +37,7 @@ regenTestnetFiles() { --totalValidators=$VALIDATOR_COUNT \ --lastUserValidator=$LAST_USER_VALIDATOR \ --outputGenesis=$NETWORK_DIR/genesis.json \ - --outputNetwork=$NETWORK_DIR/$NETWORK_FLAVOUR-network.json \ + --outputNetwork=$NETWORK_DIR/$NETWORK_TYPE-network.json \ --bootstrapAddress=$PUBLIC_IP \ --bootstrapPort=$BOOTSTRAP_PORT \ --genesisOffset=600 # Delay in seconds diff --git a/scripts/testnet0.env b/scripts/testnet0.env index a678acef56..a12a282433 100644 --- a/scripts/testnet0.env +++ b/scripts/testnet0.env @@ -1,4 +1,5 @@ NETWORK_ID=10 +NETWORK_TYPE=rlpx SHARD_COUNT=16 SLOTS_PER_EPOCH=16 SECONDS_PER_SLOT=30 diff --git a/scripts/testnet1.env b/scripts/testnet1.env index f35d9d3e9f..b02dbf75f9 100644 --- a/scripts/testnet1.env +++ b/scripts/testnet1.env @@ -1,4 +1,5 @@ NETWORK_ID=20 +NETWORK_TYPE=libp2p_spec SHARD_COUNT=16 SLOTS_PER_EPOCH=16 SECONDS_PER_SLOT=30 From 84afb77b27e4c70962e323f8cee8ea46e9957448 Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Fri, 21 Jun 2019 19:32:52 +0300 Subject: [PATCH 17/18] Restore compilation with libp2p_native after the latest changes in the spec back-end --- beacon_chain/libp2p_backend.nim | 72 ++++++++++++++++++++++++--------- 1 file changed, 54 insertions(+), 18 deletions(-) diff --git a/beacon_chain/libp2p_backend.nim b/beacon_chain/libp2p_backend.nim index c3d949e7f0..6b8665418a 100644 --- a/beacon_chain/libp2p_backend.nim +++ b/beacon_chain/libp2p_backend.nim @@ -70,7 +70,7 @@ type PeerStateInitializer* = proc(peer: Peer): RootRef {.gcsafe.} NetworkStateInitializer* = proc(network: EthereumNode): RootRef {.gcsafe.} - HandshakeStep* = proc(peer: Peer, handshakeStream: P2PStream): Future[void] {.gcsafe.} + HandshakeStep* = proc(peer: Peer, stream: P2PStream): Future[void] {.gcsafe.} DisconnectionHandler* = proc(peer: Peer): Future[void] {.gcsafe.} ThunkProc* = proc(daemon: DaemonAPI, stream: P2PStream): Future[void] {.gcsafe.} MessageContentPrinter* = proc(msg: pointer): string {.gcsafe.} @@ -96,8 +96,7 @@ proc init*(T: type Eth2Node, daemon: DaemonAPI): Future[T] {.async.} = new result result.daemon = daemon result.daemon.userData = result - result.maxInactivityAllowed = 15.minutes # TODO: Read this from the config - init result.peers + result.peers = initTable[PeerID, Peer]() newSeq result.protocolStates, allProtocols.len for proto in allProtocols: @@ -151,7 +150,7 @@ proc makeEth2Request(peer: Peer, protocolId: string, requestBytes: Bytes, # Send the request let stream = streamFut.read let sent = await stream.transp.write(requestBytes) - if sent != requestBytes: + if sent != requestBytes.len: await disconnectAndRaise(peer, FaultOrError, "Incomplete send") # Read the response @@ -222,6 +221,7 @@ proc init*(T: type Peer, network: Eth2Node, id: PeerID): Peer = result.network = network result.awaitedMessages = initTable[CompressedMsgId, FutureBase]() result.connectionState = Connected + result.maxInactivityAllowed = 15.minutes # TODO: Read this from the config newSeq result.protocolStates, allProtocols.len for i in 0 ..< allProtocols.len: let proto = allProtocols[i] @@ -236,6 +236,9 @@ proc performProtocolHandshakes*(peer: Peer) {.async.} = await all(subProtocolsHandshakes) +template initializeConnection*(peer: Peer): auto = + performProtocolHandshakes(peer) + template getRecipient(peer: Peer): Peer = peer @@ -357,7 +360,7 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = result.ResponderType = Responder result.afterProtocolInit = proc (p: P2PProtocol) = - p.onPeerConnected.params.add newIdentDefs(ident"handshakeStream", P2PStream) + p.onPeerConnected.params.add newIdentDefs(streamVar, P2PStream) result.implementMsg = proc (msg: Message) = let @@ -371,7 +374,11 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = msg.userHandler.params.insert(2, newIdentDefs(streamVar, P2PStream)) msg.initResponderCall.add streamVar - let awaitUserHandler = msg.genAwaitUserHandler(newCall("get", receivedMsg), [peerVar, streamVar]) + ## + ## Implemenmt Thunk + ## + let awaitUserHandler = msg.genAwaitUserHandler( + newCall("get", receivedMsg), [peerVar, streamVar]) let tracing = when tracingEnabled: quote do: logReceivedMsg(`streamVar`.peer, `receivedMsg`.get) @@ -382,17 +389,46 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = requestDataTimeout = newCall(milliseconds, newLit(defaultIncomingReqTimeout)) thunkName = ident(msgName & "_thunk") - msg.defineThunk quote do: - proc `thunkName`(`daemonVar`: `DaemonAPI`, `streamVar`: `P2PStream`) {.async, gcsafe.} = - var `deadlineVar` = sleepAsync `requestDataTimeout` - var `receivedMsg` = `await` readMsg(`streamVar`, `msgRecName`, `deadlineVar`) - if `receivedMsg`.isNone: - # TODO: This peer is misbehaving, perhaps we should penalize him somehow - return - let `peerVar` = `peerFromStream`(`daemonVar`, `streamVar`) - `tracing` - `awaitUserHandler` - `resolveNextMsgFutures`(`peerVar`, get(`receivedMsg`)) + msg.defineThunk if msg.kind == msgHandshake: + # In LibP2P protocols, the `onPeerConnected` handler is executed when the + # other peer opens a stream. Contrary to other thunk procs, the message is + # not immediately deserialized. Instead, the handshake "sender proc" acts + # as an exchanger that sends our handshake message while deserializing the + # contents of the other peer's handshake. + # Thus, the very first communication act of the `onPeerConnected` handler + # must be the execution of the handshake exchanger. + let handshake = msg.protocol.onPeerConnected + if handshake.isNil: + macros.error "A LibP2P protocol with a handshake must also include an " & + "`onPeerConnected` handler.", msg.procDef + + # We must generate a forward declaration for the `onPeerConnected` handler, + # so we can call it from the thunk proc: + let handshakeProcName = handshake.name + msg.protocol.outRecvProcs.add quote do: + proc `handshakeProcName`(`peerVar`: `Peer`, + `streamVar`: `P2PStream`) {.async, gcsafe.} + + quote: + proc `thunkName`(`daemonVar`: `DaemonAPI`, + `streamVar`: `P2PStream`): Future[void] {.gcsafe.} = + let `peerVar` = `peerFromStream`(`daemonVar`, `streamVar`) + return `handshakeProcName`(`peerVar`, `streamVar`) + else: + quote: + proc `thunkName`(`daemonVar`: `DaemonAPI`, + `streamVar`: `P2PStream`) {.async, gcsafe.} = + var `deadlineVar` = sleepAsync `requestDataTimeout` + var `receivedMsg` = `await` readMsg(`streamVar`, + `msgRecName`, + `deadlineVar`) + if `receivedMsg`.isNone: + # TODO: This peer is misbehaving, perhaps we should penalize him somehow + return + let `peerVar` = `peerFromStream`(`daemonVar`, `streamVar`) + `tracing` + `awaitUserHandler` + `resolveNextMsgFutures`(`peerVar`, get(`receivedMsg`)) ## ## Implement Senders and Handshake @@ -410,7 +446,7 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = handshakeExchanger.setBody quote do: let - stream = ident"handshakeStream" + stream = ident "stream" rawSendProc = `bindSymOp` `rawSendProc` params = `paramsArray` lazySendCall = newCall(rawSendProc, params) From c7512851125030cb87871c34722a9143742e375a Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Mon, 24 Jun 2019 05:34:01 +0300 Subject: [PATCH 18/18] Implement all libp2p_native response codes as specified in the latest proposal --- beacon_chain/libp2p_backend.nim | 405 +++++++++++++++--------- beacon_chain/libp2p_backends_common.nim | 13 + beacon_chain/libp2p_spec_backend.nim | 15 +- beacon_chain/sync_protocol.nim | 2 +- beacon_chain/version.nim | 2 +- 5 files changed, 263 insertions(+), 174 deletions(-) diff --git a/beacon_chain/libp2p_backend.nim b/beacon_chain/libp2p_backend.nim index 6b8665418a..5bce12e735 100644 --- a/beacon_chain/libp2p_backend.nim +++ b/beacon_chain/libp2p_backend.nim @@ -68,6 +68,12 @@ type CompressedMsgId = tuple protocolIdx, methodId: int + ResponseCode* = enum + Success + EncodingError + InvalidRequest + ServerError + PeerStateInitializer* = proc(peer: Peer): RootRef {.gcsafe.} NetworkStateInitializer* = proc(network: EthereumNode): RootRef {.gcsafe.} HandshakeStep* = proc(peer: Peer, stream: P2PStream): Future[void] {.gcsafe.} @@ -81,6 +87,8 @@ type PeerDisconnected* = object of CatchableError reason*: DisconnectionReason + TransmissionError* = object of CatchableError + const defaultIncomingReqTimeout = 5000 defaultOutgoingReqTimeout = 10000 @@ -88,9 +96,9 @@ const IrrelevantNetwork* = UselessPeer -include libp2p_backends_common include eth/p2p/p2p_backends_helpers include eth/p2p/p2p_tracing +include libp2p_backends_common proc init*(T: type Eth2Node, daemon: DaemonAPI): Future[T] {.async.} = new result @@ -107,40 +115,111 @@ proc init*(T: type Eth2Node, daemon: DaemonAPI): Future[T] {.async.} = if msg.libp2pProtocol.len > 0: await daemon.addHandler(@[msg.libp2pProtocol], msg.thunk) -proc readMsg(stream: P2PStream, MsgType: type, - deadline: Future[void]): Future[Option[MsgType]] {.async.} = +proc readMsg(stream: P2PStream, + MsgType: type, + withResponseCode: bool, + deadline: Future[void]): Future[Option[MsgType]] {.gcsafe.} + +proc readMsgBytes(stream: P2PStream, + withResponseCode: bool, + deadline: Future[void]): Future[Bytes] {.async.} = + if withResponseCode: + var responseCode: byte + var readResponseCode = stream.transp.readExactly(addr responseCode, 1) + await readResponseCode or deadline + if not readResponseCode.finished: return + if responseCode > ResponseCode.high.byte: return + + logScope: responseCode = ResponseCode(responseCode) + case ResponseCode(responseCode) + of InvalidRequest: + debug "P2P request was classified as invalid" + return + of EncodingError, ServerError: + let responseErrMsg = await readMsg(stream, string, false, deadline) + debug "P2P request resulted in error", responseErrMsg + return + of Success: + # The response is OK, the execution continues below + discard + var sizePrefix: uint32 var readSizePrefix = stream.transp.readExactly(addr sizePrefix, sizeof(sizePrefix)) await readSizePrefix or deadline if not readSizePrefix.finished: return + if sizePrefix == 0: + debug "Received SSZ with zero size", peer = stream.peer + return + var msgBytes = newSeq[byte](sizePrefix.int + sizeof(sizePrefix)) copyMem(addr msgBytes[0], addr sizePrefix, sizeof(sizePrefix)) var readBody = stream.transp.readExactly(addr msgBytes[sizeof(sizePrefix)], sizePrefix.int) await readBody or deadline if not readBody.finished: return - let decoded = SSZ.decode(msgBytes, MsgType) + return msgBytes + +proc readMsgBytesOrClose(stream: P2PStream, + withResponseCode: bool, + deadline: Future[void]): Future[Bytes] {.async.} = + result = await stream.readMsgBytes(withResponseCode, deadline) + if result.len == 0: await stream.close() + +proc readMsg(stream: P2PStream, + MsgType: type, + withResponseCode: bool, + deadline: Future[void]): Future[Option[MsgType]] {.gcsafe, async.} = + var msgBytes = await stream.readMsgBytesOrClose(withResponseCode, deadline) try: - return some(decoded) - except SerializationError: + if msgBytes.len > 0: return some SSZ.decode(msgBytes, MsgType) + except SerializationError as err: + debug "Failed to decode a network message", + msgBytes, errMsg = err.formatMsg("") return +proc sendErrorResponse(peer: Peer, + stream: P2PStream, + err: ref SerializationError, + msgName: string, + msgBytes: Bytes) {.async.} = + debug "Received an invalid request", + peer, msgName, msgBytes, errMsg = err.formatMsg("") + + var responseCode = byte(EncodingError) + discard await stream.transp.write(addr responseCode, 1) + await stream.close() + +proc sendErrorResponse(peer: Peer, + stream: P2PStream, + responseCode: ResponseCode, + errMsg: string) {.async.} = + debug "Error processing request", + peer, responseCode, errMsg + + var outputStream = init OutputStream + outputStream.append byte(responseCode) + outputStream.appendValue SSZ, errMsg + + discard await stream.transp.write(outputStream.getOutput) + await stream.close() + proc sendMsg(peer: Peer, protocolId: string, requestBytes: Bytes) {.async} = var stream = await peer.network.daemon.openStream(peer.id, @[protocolId]) # TODO how does openStream fail? Set a timeout here and handle it let sent = await stream.transp.write(requestBytes) - # TODO: Should I check that `sent` is equal to the desired number of bytes + if sent != requestBytes.len: + raise newException(TransmissionError, "Failed to deliver all bytes") proc sendBytes(stream: P2PStream, bytes: Bytes) {.async.} = let sent = await stream.transp.write(bytes) - # TODO: Should I check that `sent` is equal to the desired number of bytes + if sent != bytes.len: + raise newException(TransmissionError, "Failed to deliver all bytes") proc makeEth2Request(peer: Peer, protocolId: string, requestBytes: Bytes, ResponseMsg: type, - timeout = 10.seconds): Future[Option[ResponseMsg]] {.async.} = + timeout: Duration): Future[Option[ResponseMsg]] {.gcsafe, async.} = var deadline = sleepAsync timeout - # Open a new LibP2P stream var streamFut = peer.network.daemon.openStream(peer.id, @[protocolId]) await streamFut or deadline @@ -154,59 +233,52 @@ proc makeEth2Request(peer: Peer, protocolId: string, requestBytes: Bytes, await disconnectAndRaise(peer, FaultOrError, "Incomplete send") # Read the response - return await stream.readMsg(ResponseMsg, deadline) + return await stream.readMsg(ResponseMsg, true, deadline) + +proc exchangeHandshake(peer: Peer, protocolId: string, requestBytes: Bytes, + ResponseMsg: type, + timeout: Duration): Future[ResponseMsg] {.gcsafe, async.} = + var response = await makeEth2Request(peer, protocolId, requestBytes, + ResponseMsg, timeout) + if not response.isSome: + await peer.disconnectAndRaise(BreachOfProtocol, "Failed to complete a handshake") + + return response.get proc p2pStreamName(MsgType: type): string = mixin msgProtocol, protocolInfo, msgId MsgType.msgProtocol.protocolInfo.messages[MsgType.msgId].libp2pProtocol -template handshakeImpl(HandshakeTypeExpr: untyped, +template handshakeImpl(outputStreamVar, handshakeSerializationCall: untyped, + lowLevelThunk: untyped, + HandshakeType: untyped, # TODO: we cannot use a type parameter above # because of the following Nim issue: # - peerExpr: Peer, - streamExpr: P2PStream, - lazySendCall: Future[void], - timeoutExpr: Duration): auto = - # We make sure the inputs are evaluated only once. - let - stream = streamExpr - peer = peerExpr - timeout = timeoutExpr - - # TODO: This is a work-around for a Nim issue. Please note that it's - # semantically wrong, so if you get a compilation failure, try to - # remove it (perhaps Nim got fixed) - type HandshakeType = type(HandshakeTypeExpr) - - proc asyncStep(stream: P2PStream): Future[HandshakeType] {.async.} = - let deadline = sleepAsync timeout - var stream = stream - if stream == nil: - try: stream = await openStream(peer.network.daemon, peer.id, - @[p2pStreamName(HandshakeType)], - # TODO openStream should accept Duration - int milliseconds(timeout)) - except CatchableError: - const errMsg = "Failed to open LIBP2P stream" - debug errMsg, - stream = p2pStreamName(HandshakeType), - err = getCurrentExceptionMsg() - await disconnectAndRaise(peer, FaultOrError, errMsg) - - try: - # Please pay attention that `lazySendCall` is evaluated lazily here. - # For this reason `handshakeImpl` must remain a template. - await lazySendCall - let response = await readMsg(stream, HandshakeType, deadline) - if response.isSome: - return response.get - else: - await disconnectAndRaise(peer, BreachOfProtocol, "Handshake not completed in time") - except CatchableError: - await reraiseAsPeerDisconnected(peer, "Failure during handshake") + peer: Peer, + stream: P2PStream, + timeout: Duration): auto = + if stream == nil: + var outputStreamVar = init OutputStream + handshakeSerializationCall + exchangeHandshake(peer, p2pStreamName(HandshakeType), + getOutput(outputStreamVar), HandshakeType, timeout) + else: + proc asyncStep: Future[HandshakeType] {.async.} = + let deadline = sleepAsync timeout + var responseFut = nextMsg(peer, HandshakeType) + await lowLevelThunk(peer.network.daemon, stream) or deadline + if not responseFut.finished: + await disconnectAndRaise(peer, BreachOfProtocol, "Failed to complete a handshake") + + var outputStreamVar = init OutputStream + append(outputStreamVar, byte(Success)) + handshakeSerializationCall + await sendBytes(stream, getOutput(outputStreamVar)) - asyncStep(stream) + return responseFut.read + + asyncStep() proc resolveNextMsgFutures(peer: Peer, msg: auto) = type MsgType = type(msg) @@ -239,16 +311,6 @@ proc performProtocolHandshakes*(peer: Peer) {.async.} = template initializeConnection*(peer: Peer): auto = performProtocolHandshakes(peer) -template getRecipient(peer: Peer): Peer = - peer - -# TODO: this should be removed eventually -template getRecipient(stream: P2PStream): P2PStream = - stream - -template getRecipient(response: Responder): Peer = - UntypedResponder(response).peer - proc initProtocol(name: string, peerInit: PeerStateInitializer, networkInit: NetworkStateInitializer): ProtocolInfoObj = @@ -257,19 +319,6 @@ proc initProtocol(name: string, result.peerStateInitializer = peerInit result.networkStateInitializer = networkInit -proc registerProtocol(protocol: ProtocolInfo) = - # TODO: This can be done at compile-time in the future - let pos = lowerBound(gProtocols, protocol) - gProtocols.insert(protocol, pos) - for i in 0 ..< gProtocols.len: - gProtocols[i].index = i - -proc setEventHandlers(p: ProtocolInfo, - handshake: HandshakeStep, - disconnectHandler: DisconnectionHandler) = - p.handshake = handshake - p.disconnectHandler = disconnectHandler - proc registerMsg(protocol: ProtocolInfo, name: string, thunk: ThunkProc, @@ -281,18 +330,7 @@ proc registerMsg(protocol: ProtocolInfo, printer: printer) proc getRequestProtoName(fn: NimNode): NimNode = - when true: - return newLit("rpc/" & $fn.name) - else: - # `getCustomPragmaVal` doesn't work yet on regular nnkProcDef nodes - # (TODO: file as an issue) - let pragmas = fn.pragma - if pragmas.kind == nnkPragma and pragmas.len > 0: - for pragma in pragmas: - if pragma.len > 0 and $pragma[0] == "libp2pProtocol": - return pragma[1] - - error "All stream opening procs must have the 'libp2pProtocol' pragma specified.", fn + return newLit("/ETH/BeaconChain/" & $fn.name & "/1/SSZ") proc init*[MsgType](T: type Responder[MsgType], peer: Peer, stream: P2PStream): T = @@ -301,30 +339,41 @@ proc init*[MsgType](T: type Responder[MsgType], proc implementSendProcBody(sendProc: SendProc) = let msg = sendProc.msg - peer = sendProc.peerParam - timeout = sendProc.timeoutParam - ResponseRecord = if msg.response != nil: msg.response.recIdent else: nil UntypedResponder = bindSym "UntypedResponder" - sendMsg = bindSym "sendMsg" - sendBytes = bindSym "sendBytes" - makeEth2Request = bindSym "makeEth2Request" + await = ident "await" proc sendCallGenerator(peer, bytes: NimNode): NimNode = if msg.kind != msgResponse: let msgProto = getRequestProtoName(msg.procDef) case msg.kind of msgRequest: - let timeout = msg.timeoutParam[0] - quote: `makeEth2Request`(`peer`, `msgProto`, `bytes`, - `ResponseRecord`, `timeout`) + let + timeout = msg.timeoutParam[0] + ResponseRecord = msg.response.recIdent + quote: + makeEth2Request(`peer`, `msgProto`, `bytes`, + `ResponseRecord`, `timeout`) of msgHandshake: - quote: `sendBytes`(`peer`, `bytes`) + let + timeout = msg.timeoutParam[0] + HandshakeRecord = msg.recIdent + quote: + exchangeHandshake(`peer`, `msgProto`, `bytes`, + `HandshakeRecord`, `timeout`) else: - quote: `sendMsg`(`peer`, `msgProto`, `bytes`) + quote: sendMsg(`peer`, `msgProto`, `bytes`) else: - quote: `sendBytes`(`UntypedResponder`(`peer`).stream, `bytes`) + quote: sendBytes(`UntypedResponder`(`peer`).stream, `bytes`) + + proc prependResponseCode(stream: NimNode): NimNode = + quote: append(`stream`, byte(Success)) + + let preSerializationStep = if msg.kind == msgResponse: + prependResponseCode + else: + nil - sendProc.useStandardBody(nil, nil, sendCallGenerator) + sendProc.useStandardBody(preSerializationStep, nil, sendCallGenerator) proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = var @@ -332,20 +381,18 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = Responder = bindSym "Responder" DaemonAPI = bindSym "DaemonAPI" P2PStream = ident "P2PStream" + OutputStream = bindSym "OutputStream" Peer = bindSym "Peer" Eth2Node = bindSym "Eth2Node" messagePrinter = bindSym "messagePrinter" - peerFromStream = bindSym "peerFromStream" - handshakeImpl = bindSym "handshakeImpl" - resolveNextMsgFutures = bindSym "resolveNextMsgFutures" milliseconds = bindSym "milliseconds" registerMsg = bindSym "registerMsg" initProtocol = bindSym "initProtocol" bindSymOp = bindSym "bindSym" - receivedMsg = ident "msg" + errVar = ident "err" + msgVar = ident "msg" + msgBytesVar = ident "msgBytes" daemonVar = ident "daemon" - streamVar = ident "stream" - deadlineVar = ident "deadline" await = ident "await" p.useRequestIds = false @@ -366,6 +413,7 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = let protocol = msg.protocol msgName = $msg.ident + msgNameLit = newLit msgName msgRecName = msg.recIdent if msg.procDef.body.kind != nnkEmpty and msg.kind == msgRequest: @@ -377,27 +425,65 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = ## ## Implemenmt Thunk ## - let awaitUserHandler = msg.genAwaitUserHandler( - newCall("get", receivedMsg), [peerVar, streamVar]) + var thunkName = ident(msgName & "_thunk") + let + requestDataTimeout = newCall(milliseconds, newLit(defaultIncomingReqTimeout)) + awaitUserHandler = msg.genAwaitUserHandler(msgVar, [peerVar, streamVar]) let tracing = when tracingEnabled: - quote do: logReceivedMsg(`streamVar`.peer, `receivedMsg`.get) + quote: logReceivedMsg(`streamVar`.peer, `msgVar`.get) else: newStmtList() - let - requestDataTimeout = newCall(milliseconds, newLit(defaultIncomingReqTimeout)) - thunkName = ident(msgName & "_thunk") - - msg.defineThunk if msg.kind == msgHandshake: - # In LibP2P protocols, the `onPeerConnected` handler is executed when the - # other peer opens a stream. Contrary to other thunk procs, the message is - # not immediately deserialized. Instead, the handshake "sender proc" acts - # as an exchanger that sends our handshake message while deserializing the - # contents of the other peer's handshake. - # Thus, the very first communication act of the `onPeerConnected` handler - # must be the execution of the handshake exchanger. - let handshake = msg.protocol.onPeerConnected + msg.defineThunk quote do: + proc `thunkName`(`daemonVar`: `DaemonAPI`, + `streamVar`: `P2PStream`) {.async, gcsafe.} = + let + `deadlineVar` = sleepAsync `requestDataTimeout` + `msgBytesVar` = `await` readMsgBytes(`streamVar`, false, `deadlineVar`) + `peerVar` = peerFromStream(`daemonVar`, `streamVar`) + + if `msgBytesVar`.len == 0: + `await` sendErrorResponse(`peerVar`, `streamVar`, ServerError, + "Exceeded read timeout for a request") + return + + var `msgVar`: `msgRecName` + try: + `msgVar` = decode(`Format`, `msgBytesVar`, `msgRecName`) + except SerializationError as `errVar`: + `await` sendErrorResponse(`peerVar`, `streamVar`, `errVar`, + `msgNameLit`, `msgBytesVar`) + return + + try: + `tracing` + `awaitUserHandler` + resolveNextMsgFutures(`peerVar`, `msgVar`) + except CatchableError as `errVar`: + `await` sendErrorResponse(`peerVar`, `streamVar`, ServerError, `errVar`.msg) + + ## + ## Implement Senders and Handshake + ## + if msg.kind == msgHandshake: + # In LibP2P protocols, the handshake thunk is special. Instead of directly + # deserializing the incoming message and calling the user-supplied handler, + # we execute the `onPeerConnected` handler instead. + # + # The `onPeerConnected` handler is executed symmetrically for both peers + # and it's expected that one of its very first steps would be to send the + # handshake and then await the same from the other side. We call this step + # "handshakeExchanger". + # + # For the initiating peer, the handshakeExchanger opens a stream and sends + # a regular request through it, but on the receiving side, it just setups + # a future and call the lower-level thunk that will complete it. + # + let + handshake = msg.protocol.onPeerConnected + lowLevelThunkName = $thunkName + if handshake.isNil: macros.error "A LibP2P protocol with a handshake must also include an " & "`onPeerConnected` handler.", msg.procDef @@ -409,61 +495,64 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = proc `handshakeProcName`(`peerVar`: `Peer`, `streamVar`: `P2PStream`) {.async, gcsafe.} - quote: - proc `thunkName`(`daemonVar`: `DaemonAPI`, - `streamVar`: `P2PStream`): Future[void] {.gcsafe.} = - let `peerVar` = `peerFromStream`(`daemonVar`, `streamVar`) - return `handshakeProcName`(`peerVar`, `streamVar`) - else: - quote: + # Here we replace the 'thunkProc' that will be registered as a handler + # for incoming messages: + thunkName = ident(msgName & "_handleConnection") + + msg.protocol.outRecvProcs.add quote do: proc `thunkName`(`daemonVar`: `DaemonAPI`, `streamVar`: `P2PStream`) {.async, gcsafe.} = - var `deadlineVar` = sleepAsync `requestDataTimeout` - var `receivedMsg` = `await` readMsg(`streamVar`, - `msgRecName`, - `deadlineVar`) - if `receivedMsg`.isNone: - # TODO: This peer is misbehaving, perhaps we should penalize him somehow - return - let `peerVar` = `peerFromStream`(`daemonVar`, `streamVar`) - `tracing` - `awaitUserHandler` - `resolveNextMsgFutures`(`peerVar`, get(`receivedMsg`)) + let `peerVar` = peerFromStream(`daemonVar`, `streamVar`) + try: + `await` `handshakeProcName`(`peerVar`, `streamVar`) + except SerializationError as err: + debug "Failed to decode message", + err = err.formatMsg(""), + msg = `msgNameLit`, + peer = $(`streamVar`.peer) + `await` disconnect(`peerVar`, FaultOrError) + except CatchableError as err: + debug "Failed to complete handshake", err = err.msg + `await` disconnect(`peerVar`, FaultOrError) - ## - ## Implement Senders and Handshake - ## - var sendProc = msg.createSendProc(isRawSender = (msg.kind == msgHandshake)) - implementSendProcBody sendProc - - if msg.kind == msgHandshake: var - rawSendProc = newLit($sendProc.def.name) + handshakeSerializer = msg.createSerializer() + handshakeSerializerName = newLit($handshakeSerializer.name) handshakeExchanger = msg.createSendProc(nnkMacroDef) paramsArray = newTree(nnkBracket).appendAllParams(handshakeExchanger.def) - bindSym = ident "bindSym" + handshakeTypeName = newLit($msg.recIdent) getAst = ident "getAst" + res = ident "result" handshakeExchanger.setBody quote do: let stream = ident "stream" - rawSendProc = `bindSymOp` `rawSendProc` + outputStreamVar = ident "outputStream" + lowLevelThunk = ident `lowLevelThunkName` + HandshakeType = ident `handshakeTypeName` params = `paramsArray` - lazySendCall = newCall(rawSendProc, params) peer = params[0] timeout = params[^1] + handshakeSerializationCall = newCall(`bindSymOp` `handshakeSerializerName`, params) - lazySendCall[1] = stream - lazySendCall.del(lazySendCall.len - 1) + handshakeSerializationCall[1] = outputStreamVar + handshakeSerializationCall.del(handshakeSerializationCall.len - 1) - return `getAst`(`handshakeImpl`(`msgRecName`, peer, stream, lazySendCall, timeout)) + `res` = `getAst`(handshakeImpl(outputStreamVar, handshakeSerializationCall, + lowLevelThunk, HandshakeType, + peer, stream, timeout)) - sendProc.def.params[1][1] = P2PStream + when defined(debugMacros) or defined(debugHandshake): + echo "---- Handshake implementation ----" + echo repr(`res`) + else: + var sendProc = msg.createSendProc() + implementSendProcBody sendProc protocol.outProcRegistrations.add( newCall(registerMsg, protocol.protocolInfoVar, - newLit(msgName), + msgNameLit, thunkName, getRequestProtoName(msg.procDef), newTree(nnkBracketExpr, messagePrinter, msgRecName))) diff --git a/beacon_chain/libp2p_backends_common.nim b/beacon_chain/libp2p_backends_common.nim index e2d7353767..d88b874f8c 100644 --- a/beacon_chain/libp2p_backends_common.nim +++ b/beacon_chain/libp2p_backends_common.nim @@ -59,3 +59,16 @@ proc nextMsg*(peer: Peer, MsgType: type): Future[MsgType] = initFuture result peer.awaitedMessages[awaitedMsgId] = result +proc registerProtocol(protocol: ProtocolInfo) = + # TODO: This can be done at compile-time in the future + let pos = lowerBound(gProtocols, protocol) + gProtocols.insert(protocol, pos) + for i in 0 ..< gProtocols.len: + gProtocols[i].index = i + +proc setEventHandlers(p: ProtocolInfo, + handshake: HandshakeStep, + disconnectHandler: DisconnectionHandler) = + p.handshake = handshake + p.disconnectHandler = disconnectHandler + diff --git a/beacon_chain/libp2p_spec_backend.nim b/beacon_chain/libp2p_spec_backend.nim index 3eb4e44a6c..98084114e1 100644 --- a/beacon_chain/libp2p_spec_backend.nim +++ b/beacon_chain/libp2p_spec_backend.nim @@ -168,9 +168,9 @@ proc getThunk(protocol: ProtocolInfo, methodId: uint16): ThunkProc = if methodId.int >= protocol.messages.len: return nil protocol.messages[methodId.int].thunk -include libp2p_backends_common include eth/p2p/p2p_backends_helpers include eth/p2p/p2p_tracing +include libp2p_backends_common proc handleConnectingBeaconChainPeer(daemon: DaemonAPI, stream: P2PStream) {.async, gcsafe.} @@ -390,19 +390,6 @@ proc initProtocol(name: string, version: int, result.peerStateInitializer = peerInit result.networkStateInitializer = networkInit -proc registerProtocol(protocol: ProtocolInfo) = - # TODO: This can be done at compile-time in the future - let pos = lowerBound(gProtocols, protocol) - gProtocols.insert(protocol, pos) - for i in 0 ..< gProtocols.len: - gProtocols[i].index = i - -proc setEventHandlers(p: ProtocolInfo, - handshake: HandshakeStep, - disconnectHandler: DisconnectionHandler) = - p.handshake = handshake - p.disconnectHandler = disconnectHandler - proc registerMsg(protocol: ProtocolInfo, id: int, name: string, thunk: ThunkProc, diff --git a/beacon_chain/sync_protocol.nim b/beacon_chain/sync_protocol.nim index ecd777790c..e66e616807 100644 --- a/beacon_chain/sync_protocol.nim +++ b/beacon_chain/sync_protocol.nim @@ -138,7 +138,7 @@ p2pProtocol BeaconSync(version = 1, bestRoot: Eth2Digest, bestSlot: Slot) - proc sendGoodbye(peer: Peer, reason: DisconnectionReason) + proc goodbye(peer: Peer, reason: DisconnectionReason) requestResponse: proc getStatus( diff --git a/beacon_chain/version.nim b/beacon_chain/version.nim index b4c59263c5..dc97dc6730 100644 --- a/beacon_chain/version.nim +++ b/beacon_chain/version.nim @@ -5,7 +5,7 @@ type rlpxBackend const - network_type {.strdefine.} = "libp2p_spec" + network_type {.strdefine.} = "libp2p_native" networkBackend* = when network_type == "rlpx": rlpxBackend elif network_type == "libp2p_spec": libp2pSpecBackend