From e9dfa8642efff90d7fb7106ec234bd46f6b4fe29 Mon Sep 17 00:00:00 2001 From: subtly Date: Tue, 21 Apr 2015 05:30:41 -0400 Subject: [PATCH 01/22] compiles (v4 endpoint changes) --- libp2p/Common.cpp | 2 +- libp2p/Common.h | 11 ++++++++ libp2p/Host.cpp | 44 +++++++++++++++---------------- libp2p/NodeTable.cpp | 45 +++++++++++++------------------- libp2p/NodeTable.h | 62 +++++++++++++------------------------------- 5 files changed, 69 insertions(+), 95 deletions(-) diff --git a/libp2p/Common.cpp b/libp2p/Common.cpp index 1853679cfb1..cb39fc05138 100644 --- a/libp2p/Common.cpp +++ b/libp2p/Common.cpp @@ -24,7 +24,7 @@ using namespace std; using namespace dev; using namespace dev::p2p; -const unsigned dev::p2p::c_protocolVersion = 3; +const unsigned dev::p2p::c_protocolVersion = 4; const unsigned dev::p2p::c_defaultIPPort = 30303; const dev::p2p::NodeIPEndpoint dev::p2p::UnspecifiedNodeIPEndpoint = NodeIPEndpoint(bi::address(), 0, 0); diff --git a/libp2p/Common.h b/libp2p/Common.h index 378064e7dc0..baf90b4db27 100644 --- a/libp2p/Common.h +++ b/libp2p/Common.h @@ -36,6 +36,7 @@ #include #include #include +#include namespace ba = boost::asio; namespace bi = boost::asio::ip; @@ -162,10 +163,17 @@ using PeerSessionInfos = std::vector; */ struct NodeIPEndpoint { + enum InlineRLP + { + CreateList, + InlineList + }; + /// Setting true causes isAllowed to return true for all addresses. (Used by test fixtures) static bool test_allowLocal; NodeIPEndpoint(bi::address _addr, uint16_t _udp, uint16_t _tcp): address(_addr), udpPort(_udp), tcpPort(_tcp) {} + NodeIPEndpoint(RLP const& _r) { interpretRLP(_r); } bi::address address; uint16_t udpPort; @@ -177,6 +185,9 @@ struct NodeIPEndpoint operator bool() const { return !address.is_unspecified() && udpPort > 0 && tcpPort > 0; } bool isAllowed() const { return NodeIPEndpoint::test_allowLocal ? !address.is_unspecified() : isPublicAddress(address); } + + void streamRLP(RLPStream& _s, bool _inline = CreateList) const { if (_inline == CreateList) _s.appendList(3); if (address.is_v4()) _s << address.to_v4().to_bytes(); else _s << address.to_v6().to_bytes(); _s << udpPort << tcpPort; } + void interpretRLP(RLP const& _r) { if (_r[0].size() == 4) address = bi::address_v4(_r[0].toArray()); else address = bi::address_v6(_r[0].toArray()); udpPort = _r[1].toInt(); tcpPort = _r[2].toInt(); } }; struct Node diff --git a/libp2p/Host.cpp b/libp2p/Host.cpp index fff71829568..08289bea8de 100644 --- a/libp2p/Host.cpp +++ b/libp2p/Host.cpp @@ -678,15 +678,16 @@ bytes Host::saveNetwork() const int count = 0; for (auto const& p: peers) { - // Only save peers which have connected within 2 days, with properly-advertised port and public IP address - // todo: e2e ipv6 support + // todo: ipv6 if (!p.endpoint.address.is_v4()) continue; - if (chrono::system_clock::now() - p.m_lastConnected < chrono::seconds(3600 * 48) && p.endpoint.tcpPort > 0 && p.id != id() && (p.required || p.endpoint.isAllowed())) + // Only save peers which have connected within 2 days, with properly-advertised port and public IP address + if (chrono::system_clock::now() - p.m_lastConnected < chrono::seconds(3600 * 48) && !!p.endpoint && p.id != id() && (p.required || p.endpoint.isAllowed())) { - network.appendList(10); - network << p.endpoint.address.to_v4().to_bytes() << p.endpoint.tcpPort << p.id << p.required + network.appendList(11); + p.endpoint.streamRLP(network, NodeIPEndpoint::InlineList); + network << p.id << p.required << chrono::duration_cast(p.m_lastConnected.time_since_epoch()).count() << chrono::duration_cast(p.m_lastAttempted.time_since_epoch()).count() << p.m_failedAttempts << (unsigned)p.m_lastDisconnect << p.m_score << p.m_rating; @@ -700,12 +701,9 @@ bytes Host::saveNetwork() const state.sort(); for (auto const& entry: state) { - network.appendList(3); - if (entry.endpoint.address.is_v4()) - network << entry.endpoint.address.to_v4().to_bytes(); - else - network << entry.endpoint.address.to_v6().to_bytes(); - network << entry.endpoint.tcpPort << entry.id; + network.appendList(4); + entry.endpoint.streamRLP(network, NodeIPEndpoint::InlineList); + network << entry.id; count++; } } @@ -738,25 +736,25 @@ void Host::restoreNetwork(bytesConstRef _b) for (auto i: r[2]) { + // todo: ipv6 if (i[0].itemCount() != 4) continue; - - // todo: ipv6, bi::address_v6(i[0].toArray() - Node n((NodeId)i[2], NodeIPEndpoint(bi::address_v4(i[0].toArray()), i[1].toInt(), i[1].toInt())); - if (i.itemCount() == 3 && n.endpoint.isAllowed()) + + Node n((NodeId)i[3], NodeIPEndpoint(i)); + if (i.itemCount() == 4 && n.endpoint.isAllowed()) m_nodeTable->addNode(n); - else if (i.itemCount() == 10) + else if (i.itemCount() == 11) { - n.required = i[3].toInt(); + n.required = i[4].toInt(); if (!n.endpoint.isAllowed() && !n.required) continue; shared_ptr p = make_shared(n); - p->m_lastConnected = chrono::system_clock::time_point(chrono::seconds(i[4].toInt())); - p->m_lastAttempted = chrono::system_clock::time_point(chrono::seconds(i[5].toInt())); - p->m_failedAttempts = i[6].toInt(); - p->m_lastDisconnect = (DisconnectReason)i[7].toInt(); - p->m_score = (int)i[8].toInt(); - p->m_rating = (int)i[9].toInt(); + p->m_lastConnected = chrono::system_clock::time_point(chrono::seconds(i[5].toInt())); + p->m_lastAttempted = chrono::system_clock::time_point(chrono::seconds(i[6].toInt())); + p->m_failedAttempts = i[7].toInt(); + p->m_lastDisconnect = (DisconnectReason)i[8].toInt(); + p->m_score = (int)i[9].toInt(); + p->m_rating = (int)i[10].toInt(); m_peers[p->id] = p; if (p->required) requirePeer(p->id, n.endpoint); diff --git a/libp2p/NodeTable.cpp b/libp2p/NodeTable.cpp index 35db7fbac60..e7f4d1f33b8 100644 --- a/libp2p/NodeTable.cpp +++ b/libp2p/NodeTable.cpp @@ -99,9 +99,7 @@ shared_ptr NodeTable::addNode(Node const& _node) Guard l(x_pubkDiscoverPings); m_pubkDiscoverPings[_node.endpoint.address] = std::chrono::steady_clock::now(); } - PingNode p(_node.endpoint, m_node.endpoint.address.to_string(), m_node.endpoint.udpPort); - p.sign(m_secret); - m_socketPointer->send(p); + ping(_node.endpoint); return move(shared_ptr()); } @@ -114,9 +112,7 @@ shared_ptr NodeTable::addNode(Node const& _node) shared_ptr ret(new NodeEntry(m_node, _node.id, _node.endpoint)); m_nodes[_node.id] = ret; clog(NodeTableConnect) << "addNode pending for" << _node.endpoint; - PingNode p(_node.endpoint, m_node.endpoint.address.to_string(), m_node.endpoint.udpPort); - p.sign(m_secret); - m_socketPointer->send(p); + ping(_node.endpoint); return ret; } @@ -291,9 +287,9 @@ vector> NodeTable::nearestNodeEntries(NodeId _target) return move(ret); } -void NodeTable::ping(bi::udp::endpoint _to) const +void NodeTable::ping(NodeIPEndpoint _to) const { - PingNode p(_to, m_node.endpoint.address.to_string(), m_node.endpoint.udpPort); + PingNode p(m_node.endpoint, _to); p.sign(m_secret); m_socketPointer->send(p); } @@ -488,8 +484,8 @@ void NodeTable::onReceived(UDPSocketFace*, bi::udp::endpoint const& _from, bytes } Neighbours in = Neighbours::fromBytesConstRef(_from, rlpBytes); - for (auto n: in.nodes) - addNode(n.node, NodeIPEndpoint(bi::address::from_string(n.ipAddress), n.udpPort, n.udpPort)); + for (auto n: in.neighbours) + addNode(n.node, n.endpoint); break; } @@ -520,10 +516,9 @@ void NodeTable::onReceived(UDPSocketFace*, bi::udp::endpoint const& _from, bytes return; } - // TODO: Feedback if _from.address() != in.ipAddress - addNode(nodeid, NodeIPEndpoint(_from.address(), _from.port(), in.tcpPort)); - - Pong p(_from); + // TODO: Feedback if _from.address() != in.ipAddress, or, _from.por() != in.source.udpPort + auto node = addNode(nodeid, NodeIPEndpoint(_from.address(), _from.port(), in.source.tcpPort)); + Pong p(node->endpoint); p.echo = sha3(rlpBytes); p.sign(m_secret); m_socketPointer->send(p); @@ -599,26 +594,22 @@ void NodeTable::doRefreshBuckets(boost::system::error_code const& _ec) void PingNode::streamRLP(RLPStream& _s) const { _s.appendList(4); - _s << dev::p2p::c_protocolVersion << ipAddress << tcpPort << ts; + _s << dev::p2p::c_protocolVersion; + source.streamRLP(_s); + destination.streamRLP(_s); + _s << ts; } void PingNode::interpretRLP(bytesConstRef _bytes) { RLP r(_bytes); - if (r.itemCountStrict() == 3) - { - version = 2; - ipAddress = r[0].toString(); - tcpPort = r[1].toInt(RLP::Strict); - ts = r[2].toInt(RLP::Strict); - } - else if (r.itemCountStrict() == 4) + if (r.itemCountStrict() == 4 && r[0].isInt() && r[0].toInt(RLP::Strict) == dev::p2p::c_protocolVersion) { - version = r[0].toInt(RLP::Strict); - ipAddress = r[1].toString(); - tcpPort = r[2].toInt(RLP::Strict); + version = dev::p2p::c_protocolVersion; + source.interpretRLP(r[1]); + destination.interpretRLP(r[2]); ts = r[3].toInt(RLP::Strict); } else - BOOST_THROW_EXCEPTION(InvalidRLP()); + version = 0; } diff --git a/libp2p/NodeTable.h b/libp2p/NodeTable.h index 07247462f3e..c96abde86fb 100644 --- a/libp2p/NodeTable.h +++ b/libp2p/NodeTable.h @@ -204,7 +204,7 @@ class NodeTable: UDPSocketEvents, public std::enable_shared_from_this }; /// Used to ping endpoint. - void ping(bi::udp::endpoint _to) const; + void ping(NodeIPEndpoint _to) const; /// Used ping known node. Used by node table when refreshing buckets and as part of eviction process (see evict). void ping(NodeEntry* _n) const; @@ -299,29 +299,17 @@ struct InvalidRLP: public Exception {}; * a given bucket which is full, the least-responsive node is pinged. * If the pinged node doesn't respond, then it is removed and the new * node is inserted. - * - * RLP Encoded Items: 3 - * Minimum Encoded Size: 18 bytes - * Maximum Encoded Size: bytes // todo after u128 addresses - * - * signature: Signature of message. - * ipAddress: Our IP address. - * port: Our port. - * - * @todo uint128_t for ip address (<->integer ipv4/6, asio-address, asio-endpoint) - * */ struct PingNode: RLPXDatagram { - PingNode(bi::udp::endpoint _ep): RLPXDatagram(_ep) {} - PingNode(bi::udp::endpoint _ep, std::string _src, uint16_t _srcPort, std::chrono::seconds _ts = std::chrono::seconds(60)): RLPXDatagram(_ep), ipAddress(_src), tcpPort(_srcPort), ts(futureFromEpoch(_ts)) {} + PingNode(bi::udp::endpoint _ep): RLPXDatagram(_ep), source(UnspecifiedNodeIPEndpoint), destination(UnspecifiedNodeIPEndpoint) {} + PingNode(NodeIPEndpoint _src, NodeIPEndpoint _dest): RLPXDatagram(_dest), source(_src), destination(_dest), ts(futureFromEpoch(std::chrono::seconds(60))) {} static const uint8_t type = 1; unsigned version = 0; - std::string ipAddress; -// uint16_t udpPort; - uint16_t tcpPort; + NodeIPEndpoint source; + NodeIPEndpoint destination; unsigned ts; void streamRLP(RLPStream& _s) const override; @@ -330,17 +318,15 @@ struct PingNode: RLPXDatagram /** * Pong packet: Sent in response to ping - * - * RLP Encoded Items: 2 - * Minimum Encoded Size: 33 bytes - * Maximum Encoded Size: 33 bytes */ struct Pong: RLPXDatagram { - Pong(bi::udp::endpoint _ep): RLPXDatagram(_ep), ts(futureFromEpoch(std::chrono::seconds(60))) {} + Pong(bi::udp::endpoint _ep): RLPXDatagram(_ep), destination(UnspecifiedNodeIPEndpoint) {} + Pong(NodeIPEndpoint _dest): RLPXDatagram((bi::udp::endpoint)_dest), destination(_dest), ts(futureFromEpoch(std::chrono::seconds(60))) {} static const uint8_t type = 2; + NodeIPEndpoint destination; h256 echo; ///< MCD of PingNode unsigned ts; @@ -375,23 +361,17 @@ struct FindNode: RLPXDatagram }; /** - * Node Packet: Multiple node packets are sent in response to FindNode. - * - * RLP Encoded Items: 2 (first item is list) - * Minimum Encoded Size: 10 bytes + * Node Packet: One or more node packets are sent in response to FindNode. */ struct Neighbours: RLPXDatagram { - struct Node + struct Neighbour { - Node() = default; - Node(RLP const& _r) { interpretRLP(_r); } - std::string ipAddress; - uint16_t udpPort; -// uint16_t tcpPort; + Neighbour(Node const& _node): endpoint(_node.endpoint), node(_node.id) {} + Neighbour(RLP const& _r): endpoint(_r) { node = h512(_r[3].toBytes()); } + NodeIPEndpoint endpoint; NodeId node; - void streamRLP(RLPStream& _s) const { _s.appendList(3); _s << ipAddress << udpPort << node; } - void interpretRLP(RLP const& _r) { ipAddress = _r[0].toString(); udpPort = _r[1].toInt(); node = h512(_r[2].toBytes()); } + void streamRLP(RLPStream& _s) const { _s.appendList(4); endpoint.streamRLP(_s, NodeIPEndpoint::InlineList); _s << node; } }; Neighbours(bi::udp::endpoint _ep): RLPXDatagram(_ep), ts(futureFromEpoch(std::chrono::seconds(30))) {} @@ -399,21 +379,15 @@ struct Neighbours: RLPXDatagram { auto limit = _limit ? std::min(_nearest.size(), (size_t)(_offset + _limit)) : _nearest.size(); for (auto i = _offset; i < limit; i++) - { - Node node; - node.ipAddress = _nearest[i]->endpoint.address.to_string(); - node.udpPort = _nearest[i]->endpoint.udpPort; - node.node = _nearest[i]->publicKey(); - nodes.push_back(node); - } + neighbours.push_back(Neighbour(*_nearest[i])); } static const uint8_t type = 4; - std::vector nodes; + std::vector neighbours; unsigned ts = 1; - void streamRLP(RLPStream& _s) const { _s.appendList(2); _s.appendList(nodes.size()); for (auto& n: nodes) n.streamRLP(_s); _s << ts; } - void interpretRLP(bytesConstRef _bytes) { RLP r(_bytes); for (auto n: r[0]) nodes.push_back(Node(n)); ts = r[1].toInt(); } + void streamRLP(RLPStream& _s) const { _s.appendList(2); _s.appendList(neighbours.size()); for (auto& n: neighbours) n.streamRLP(_s); _s << ts; } + void interpretRLP(bytesConstRef _bytes) { RLP r(_bytes); for (auto n: r[0]) neighbours.push_back(Neighbour(n)); ts = r[1].toInt(); } }; struct NodeTableWarn: public LogChannel { static const char* name(); static const int verbosity = 0; }; From 0850c7ac72a0273b6f94de5607bc17dce5b730a1 Mon Sep 17 00:00:00 2001 From: subtly Date: Tue, 21 Apr 2015 05:31:33 -0400 Subject: [PATCH 02/22] update tests (v4 endpoint) --- test/libp2p/net.cpp | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/test/libp2p/net.cpp b/test/libp2p/net.cpp index 9a5dbb32f7b..de599521cfd 100644 --- a/test/libp2p/net.cpp +++ b/test/libp2p/net.cpp @@ -82,7 +82,7 @@ struct TestNodeTable: public NodeTable bi::address ourIp = bi::address::from_string("127.0.0.1"); for (auto& n: _testNodes) { - ping(bi::udp::endpoint(ourIp, n.second)); + ping(NodeIPEndpoint(ourIp, n.second, n.second)); this_thread::sleep_for(chrono::milliseconds(2)); } } @@ -244,11 +244,9 @@ BOOST_AUTO_TEST_CASE(neighboursPacketLength) auto limit = nlimit ? std::min(testNodes.size(), (size_t)(offset + nlimit)) : testNodes.size(); for (auto i = offset; i < limit; i++) { - Neighbours::Node node; - node.ipAddress = boost::asio::ip::address::from_string("200.200.200.200").to_string(); - node.udpPort = testNodes[i].second; - node.node = testNodes[i].first.pub(); - out.nodes.push_back(node); + Node n(testNodes[i].first.pub(), NodeIPEndpoint(boost::asio::ip::address::from_string("200.200.200.200"), testNodes[i].second, testNodes[i].second)); + Neighbours::Neighbour neighbour(n); + out.neighbours.push_back(neighbour); } out.sign(k.sec()); @@ -265,11 +263,9 @@ BOOST_AUTO_TEST_CASE(test_neighbours_packet) Neighbours out(to); for (auto n: testNodes) { - Neighbours::Node node; - node.ipAddress = boost::asio::ip::address::from_string("127.0.0.1").to_string(); - node.udpPort = n.second; - node.node = n.first.pub(); - out.nodes.push_back(node); + Node node(n.first.pub(), NodeIPEndpoint(boost::asio::ip::address::from_string("200.200.200.200"), n.second, n.second)); + Neighbours::Neighbour neighbour(node); + out.neighbours.push_back(neighbour); } out.sign(k.sec()); @@ -277,9 +273,9 @@ BOOST_AUTO_TEST_CASE(test_neighbours_packet) bytesConstRef rlpBytes(packet.cropped(h256::size + Signature::size + 1)); Neighbours in = Neighbours::fromBytesConstRef(to, rlpBytes); int count = 0; - for (auto n: in.nodes) + for (auto n: in.neighbours) { - BOOST_REQUIRE_EQUAL(testNodes[count].second, n.udpPort); + BOOST_REQUIRE_EQUAL(testNodes[count].second, n.endpoint.udpPort); BOOST_REQUIRE_EQUAL(testNodes[count].first.pub(), n.node); BOOST_REQUIRE_EQUAL(sha3(testNodes[count].first.pub()), sha3(n.node)); count++; From c70f21acec1a08ae7ab46da952ac360b119011ff Mon Sep 17 00:00:00 2001 From: subtly Date: Tue, 21 Apr 2015 05:38:39 -0400 Subject: [PATCH 03/22] update net tests (v4 endpoint) --- test/libp2p/net.cpp | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/test/libp2p/net.cpp b/test/libp2p/net.cpp index de599521cfd..f2808364e61 100644 --- a/test/libp2p/net.cpp +++ b/test/libp2p/net.cpp @@ -226,7 +226,7 @@ BOOST_AUTO_TEST_CASE(v2PingNodePacket) PingNode p((bi::udp::endpoint())); BOOST_REQUIRE_NO_THROW(p = PingNode::fromBytesConstRef(bi::udp::endpoint(), bytesConstRef(&s.out()))); - BOOST_REQUIRE(p.version == 2); + BOOST_REQUIRE(p.version == 0); } BOOST_AUTO_TEST_CASE(neighboursPacketLength) @@ -254,7 +254,7 @@ BOOST_AUTO_TEST_CASE(neighboursPacketLength) } } -BOOST_AUTO_TEST_CASE(test_neighbours_packet) +BOOST_AUTO_TEST_CASE(neighboursPacket) { KeyPair k = KeyPair::create(); std::vector> testNodes(TestNodeTable::createTestNodes(16)); @@ -289,12 +289,6 @@ BOOST_AUTO_TEST_CASE(test_findnode_neighbours) // into the same list of nearest nodes. } -BOOST_AUTO_TEST_CASE(test_windows_template) -{ - bi::udp::endpoint ep; - PingNode p(ep); -} - BOOST_AUTO_TEST_CASE(kademlia) { // Not yet a 'real' test. @@ -328,7 +322,7 @@ BOOST_AUTO_TEST_CASE(kademlia) } -BOOST_AUTO_TEST_CASE(test_udp_once) +BOOST_AUTO_TEST_CASE(udpOnce) { UDPDatagram d(bi::udp::endpoint(boost::asio::ip::address::from_string("127.0.0.1"), 30300), bytes({65,65,65,65})); TestUDPSocket a; a.m_socket->connect(); a.start(); From d76dfef797ee460a40f4d46c6a9619401b7eac42 Mon Sep 17 00:00:00 2001 From: subtly Date: Tue, 21 Apr 2015 06:13:31 -0400 Subject: [PATCH 04/22] update peer host test. support unspecified ip. --- libp2p/Common.h | 2 +- libp2p/NodeTable.cpp | 3 ++- test/libp2p/peer.cpp | 4 +++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/libp2p/Common.h b/libp2p/Common.h index baf90b4db27..54dfc8f9d90 100644 --- a/libp2p/Common.h +++ b/libp2p/Common.h @@ -187,7 +187,7 @@ struct NodeIPEndpoint bool isAllowed() const { return NodeIPEndpoint::test_allowLocal ? !address.is_unspecified() : isPublicAddress(address); } void streamRLP(RLPStream& _s, bool _inline = CreateList) const { if (_inline == CreateList) _s.appendList(3); if (address.is_v4()) _s << address.to_v4().to_bytes(); else _s << address.to_v6().to_bytes(); _s << udpPort << tcpPort; } - void interpretRLP(RLP const& _r) { if (_r[0].size() == 4) address = bi::address_v4(_r[0].toArray()); else address = bi::address_v6(_r[0].toArray()); udpPort = _r[1].toInt(); tcpPort = _r[2].toInt(); } + void interpretRLP(RLP const& _r) { if (_r[0].size() == 0) address = bi::address(); else if (_r[0].size() == 4) address = bi::address_v4(_r[0].toArray()); else address = bi::address_v6(_r[0].toArray()); udpPort = _r[1].toInt(); tcpPort = _r[2].toInt(); } }; struct Node diff --git a/libp2p/NodeTable.cpp b/libp2p/NodeTable.cpp index e7f4d1f33b8..6d348364014 100644 --- a/libp2p/NodeTable.cpp +++ b/libp2p/NodeTable.cpp @@ -512,7 +512,8 @@ void NodeTable::onReceived(UDPSocketFace*, bi::udp::endpoint const& _from, bytes if (in.version != dev::p2p::c_protocolVersion) { if (auto n = nodeEntry(nodeid)) - dropNode(n); + if (n) + dropNode(n); return; } diff --git a/test/libp2p/peer.cpp b/test/libp2p/peer.cpp index 727430fc819..7701d214cef 100644 --- a/test/libp2p/peer.cpp +++ b/test/libp2p/peer.cpp @@ -51,9 +51,11 @@ BOOST_AUTO_TEST_CASE(host) auto node2 = host2.id(); host2.start(); + while (!host2.isStarted()) + this_thread::sleep_for(chrono::milliseconds(20)); host1.addNode(node2, NodeIPEndpoint(bi::address::from_string("127.0.0.1"), host2prefs.listenPort, host2prefs.listenPort)); - this_thread::sleep_for(chrono::seconds(3)); + this_thread::sleep_for(chrono::seconds(10)); auto host1peerCount = host1.peerCount(); auto host2peerCount = host2.peerCount(); From c71d43178380315863c9b9899ba1554201d57a76 Mon Sep 17 00:00:00 2001 From: subtly Date: Tue, 21 Apr 2015 18:13:45 -0400 Subject: [PATCH 05/22] more test updates (v4 endpoints. #1557, #1558) --- test/libp2p/peer.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/libp2p/peer.cpp b/test/libp2p/peer.cpp index 7701d214cef..0bdea03055a 100644 --- a/test/libp2p/peer.cpp +++ b/test/libp2p/peer.cpp @@ -74,7 +74,7 @@ BOOST_AUTO_TEST_CASE(networkConfig) BOOST_REQUIRE(save.id() == restore.id()); } -BOOST_AUTO_TEST_CASE(save_nodes) +BOOST_AUTO_TEST_CASE(saveNodes) { std::list hosts; for (auto i:{0,1,2,3,4,5}) @@ -113,7 +113,7 @@ BOOST_AUTO_TEST_CASE(save_nodes) for (auto i: r[2]) { - BOOST_REQUIRE(i.itemCount() == 3 || i.itemCount() == 10); + BOOST_REQUIRE(i.itemCount() == 4 || i.itemCount() == 11); BOOST_REQUIRE(i[0].itemCount() == 4 || i[0].itemCount() == 16); } } From 50f5c96e9ca6abfe897fb9f92b3828c109bccbc1 Mon Sep 17 00:00:00 2001 From: subtly Date: Thu, 23 Apr 2015 03:04:35 +0100 Subject: [PATCH 06/22] handle new endpoint properties for ping --- libp2p/NodeTable.cpp | 96 ++++++++++++++++++++++---------------------- libp2p/NodeTable.h | 9 ++--- test/libp2p/peer.cpp | 2 +- 3 files changed, 53 insertions(+), 54 deletions(-) diff --git a/libp2p/NodeTable.cpp b/libp2p/NodeTable.cpp index 0c1beef040a..3370e502b26 100644 --- a/libp2p/NodeTable.cpp +++ b/libp2p/NodeTable.cpp @@ -75,21 +75,10 @@ void NodeTable::processEvents() m_nodeEventHandler->processEvents(); } -shared_ptr NodeTable::addNode(Public const& _pubk, NodeIPEndpoint const& _ep) -{ - auto node = Node(_pubk, _ep); - return addNode(node); -} - shared_ptr NodeTable::addNode(Node const& _node) { - // re-enable tcp checks when NAT hosts are handled by discover - // we handle when tcp endpoint is 0 below - if (_node.endpoint.address.to_string() == "0.0.0.0") - { - clog(NodeTableWarn) << "addNode Failed. Invalid UDP address" << url << "0.0.0.0" << "for" << _node.id; + if (!_node.endpoint) return move(shared_ptr()); - } // ping address to recover nodeid if nodeid is empty if (!_node.id) @@ -229,22 +218,24 @@ vector> NodeTable::nearestNodeEntries(NodeId _target) Guard l(x_state); for (auto n: m_state[head].nodes) if (auto p = n.lock()) - { - if (count < s_bucketSize) - found[distance(_target, p->id)].push_back(p); - else - break; - } - - if (count < s_bucketSize && tail) - for (auto n: m_state[tail].nodes) - if (auto p = n.lock()) + if (!!p->endpoint && p->endpoint.isAllowed()) { if (count < s_bucketSize) found[distance(_target, p->id)].push_back(p); else break; } + + if (count < s_bucketSize && tail) + for (auto n: m_state[tail].nodes) + if (auto p = n.lock()) + if (!!p->endpoint && p->endpoint.isAllowed()) + { + if (count < s_bucketSize) + found[distance(_target, p->id)].push_back(p); + else + break; + } head++; if (tail) @@ -256,12 +247,13 @@ vector> NodeTable::nearestNodeEntries(NodeId _target) Guard l(x_state); for (auto n: m_state[head].nodes) if (auto p = n.lock()) - { - if (count < s_bucketSize) - found[distance(_target, p->id)].push_back(p); - else - break; - } + if (!!p->endpoint && p->endpoint.isAllowed()) + { + if (count < s_bucketSize) + found[distance(_target, p->id)].push_back(p); + else + break; + } head++; } else @@ -270,19 +262,20 @@ vector> NodeTable::nearestNodeEntries(NodeId _target) Guard l(x_state); for (auto n: m_state[tail].nodes) if (auto p = n.lock()) - { - if (count < s_bucketSize) - found[distance(_target, p->id)].push_back(p); - else - break; - } + if (!!p->endpoint && p->endpoint.isAllowed()) + { + if (count < s_bucketSize) + found[distance(_target, p->id)].push_back(p); + else + break; + } tail--; } vector> ret; for (auto& nodes: found) for (auto n: nodes.second) - if (n->endpoint.isAllowed()) + if (ret.size() < s_bucketSize && n->endpoint.isAllowed()) ret.push_back(n); return move(ret); } @@ -320,8 +313,9 @@ void NodeTable::noteActiveNode(Public const& _pubk, bi::udp::endpoint const& _en return; shared_ptr node = nodeEntry(_pubk); - if (!!node && !node->pending) + if (!!node && !node->pending && !!node->endpoint) { + // todo: drop in favor of ping/pong packets clog(NodeTableConnect) << "Noting active node:" << _pubk.abridged() << _endpoint.address().to_string() << ":" << _endpoint.port(); node->endpoint.address = _endpoint.address(); node->endpoint.udpPort = _endpoint.port(); @@ -426,6 +420,8 @@ void NodeTable::onReceived(UDPSocketFace*, bi::udp::endpoint const& _from, bytes { Pong in = Pong::fromBytesConstRef(_from, rlpBytes); + // TODO: check echo! (pending pings) + // whenever a pong is received, check if it's in m_evictions Guard le(x_evictions); bool evictionEntry = false; @@ -454,12 +450,22 @@ void NodeTable::onReceived(UDPSocketFace*, bi::udp::endpoint const& _from, bytes m_pubkDiscoverPings.erase(_from.address()); } if (!haveNode(nodeid)) - addNode(nodeid, NodeIPEndpoint(_from.address(), _from.port(), _from.port())); + addNode(Node(nodeid, NodeIPEndpoint(_from.address(), _from.port(), _from.port()))); } else return; // unsolicited pong; don't note node as active } + // update our endpoint address and UDP port (with caution and iff appropriate) + if (false && !m_node.endpoint) + { + if (in.destination.address != m_node.endpoint.address) + m_node.endpoint.address = in.destination.address; + + if (in.destination.udpPort != m_node.endpoint.udpPort) + m_node.endpoint.udpPort = in.destination.udpPort; + } + clog(NodeTableConnect) << "PONG from " << nodeid.abridged() << _from; break; } @@ -485,7 +491,7 @@ void NodeTable::onReceived(UDPSocketFace*, bi::udp::endpoint const& _from, bytes Neighbours in = Neighbours::fromBytesConstRef(_from, rlpBytes); for (auto n: in.neighbours) - addNode(n.node, n.endpoint); + addNode(Node(n.node, n.endpoint)); break; } @@ -510,16 +516,12 @@ void NodeTable::onReceived(UDPSocketFace*, bi::udp::endpoint const& _from, bytes { PingNode in = PingNode::fromBytesConstRef(_from, rlpBytes); if (in.version != dev::p2p::c_protocolVersion) - { - if (auto n = nodeEntry(nodeid)) - if (n) - dropNode(n); return; - } - - // TODO: Feedback if _from.address() != in.ipAddress, or, _from.por() != in.source.udpPort - auto node = addNode(nodeid, NodeIPEndpoint(_from.address(), _from.port(), in.source.tcpPort)); - Pong p(node->endpoint); + + if (in.source.address.is_unspecified()) + in.source.address = _from.address(); + addNode(Node(nodeid, in.source)); + Pong p(NodeIPEndpoint(_from.address(), _from.port(), in.source.tcpPort)); p.echo = sha3(rlpBytes); p.sign(m_secret); m_socketPointer->send(p); diff --git a/libp2p/NodeTable.h b/libp2p/NodeTable.h index c96abde86fb..4bb9a2178b8 100644 --- a/libp2p/NodeTable.h +++ b/libp2p/NodeTable.h @@ -146,10 +146,7 @@ class NodeTable: UDPSocketEvents, public std::enable_shared_from_this /// Called by implementation which provided handler to process NodeEntryAdded/NodeEntryDropped events. Events are coalesced by type whereby old events are ignored. void processEvents(); - /// Add node. Node will be pinged and empty shared_ptr is returned if NodeId is uknown. - std::shared_ptr addNode(Public const& _pubk, NodeIPEndpoint const& _ep); - - /// Add node. Node will be pinged and empty shared_ptr is returned if node has never been seen. + /// Add node. Node will be pinged and empty shared_ptr is returned if node has never been seen or NodeId is empty. std::shared_ptr addNode(Node const& _node); /// To be called when node table is empty. Runs node discovery with m_node.id as the target in order to populate node-table. @@ -321,8 +318,8 @@ struct PingNode: RLPXDatagram */ struct Pong: RLPXDatagram { - Pong(bi::udp::endpoint _ep): RLPXDatagram(_ep), destination(UnspecifiedNodeIPEndpoint) {} - Pong(NodeIPEndpoint _dest): RLPXDatagram((bi::udp::endpoint)_dest), destination(_dest), ts(futureFromEpoch(std::chrono::seconds(60))) {} + Pong(bi::udp::endpoint const& _ep): RLPXDatagram(_ep), destination(UnspecifiedNodeIPEndpoint) {} + Pong(NodeIPEndpoint const& _dest): RLPXDatagram((bi::udp::endpoint)_dest), destination(_dest), ts(futureFromEpoch(std::chrono::seconds(60))) {} static const uint8_t type = 2; diff --git a/test/libp2p/peer.cpp b/test/libp2p/peer.cpp index 0bdea03055a..7532cda0ecd 100644 --- a/test/libp2p/peer.cpp +++ b/test/libp2p/peer.cpp @@ -55,7 +55,7 @@ BOOST_AUTO_TEST_CASE(host) this_thread::sleep_for(chrono::milliseconds(20)); host1.addNode(node2, NodeIPEndpoint(bi::address::from_string("127.0.0.1"), host2prefs.listenPort, host2prefs.listenPort)); - this_thread::sleep_for(chrono::seconds(10)); + this_thread::sleep_for(chrono::seconds(3)); auto host1peerCount = host1.peerCount(); auto host2peerCount = host2.peerCount(); From b3f4e7777be4155518e9d2c1bb7feb89c4ee371e Mon Sep 17 00:00:00 2001 From: subtly Date: Thu, 23 Apr 2015 17:58:52 +0100 Subject: [PATCH 07/22] Minor semantic updates. Support unspecified address for NodeIPEndpoint. --- libp2p/Common.h | 8 ++++---- libp2p/Host.cpp | 4 ++-- libp2p/NodeTable.h | 6 ++++-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/libp2p/Common.h b/libp2p/Common.h index 54dfc8f9d90..0b572473760 100644 --- a/libp2p/Common.h +++ b/libp2p/Common.h @@ -163,10 +163,10 @@ using PeerSessionInfos = std::vector; */ struct NodeIPEndpoint { - enum InlineRLP + enum RLPAppend { - CreateList, - InlineList + List, + Inline }; /// Setting true causes isAllowed to return true for all addresses. (Used by test fixtures) @@ -186,7 +186,7 @@ struct NodeIPEndpoint bool isAllowed() const { return NodeIPEndpoint::test_allowLocal ? !address.is_unspecified() : isPublicAddress(address); } - void streamRLP(RLPStream& _s, bool _inline = CreateList) const { if (_inline == CreateList) _s.appendList(3); if (address.is_v4()) _s << address.to_v4().to_bytes(); else _s << address.to_v6().to_bytes(); _s << udpPort << tcpPort; } + void streamRLP(RLPStream& _s, RLPAppend _inline = List) const { if (_inline == List) _s.appendList(3); if (address.is_v4()) _s << address.to_v4().to_bytes(); else if (address.is_v6()) _s << address.to_v6().to_bytes(); else _s << ""; _s << udpPort << tcpPort; } void interpretRLP(RLP const& _r) { if (_r[0].size() == 0) address = bi::address(); else if (_r[0].size() == 4) address = bi::address_v4(_r[0].toArray()); else address = bi::address_v6(_r[0].toArray()); udpPort = _r[1].toInt(); tcpPort = _r[2].toInt(); } }; diff --git a/libp2p/Host.cpp b/libp2p/Host.cpp index 08289bea8de..aea51ddd000 100644 --- a/libp2p/Host.cpp +++ b/libp2p/Host.cpp @@ -686,7 +686,7 @@ bytes Host::saveNetwork() const if (chrono::system_clock::now() - p.m_lastConnected < chrono::seconds(3600 * 48) && !!p.endpoint && p.id != id() && (p.required || p.endpoint.isAllowed())) { network.appendList(11); - p.endpoint.streamRLP(network, NodeIPEndpoint::InlineList); + p.endpoint.streamRLP(network, NodeIPEndpoint::Inline); network << p.id << p.required << chrono::duration_cast(p.m_lastConnected.time_since_epoch()).count() << chrono::duration_cast(p.m_lastAttempted.time_since_epoch()).count() @@ -702,7 +702,7 @@ bytes Host::saveNetwork() const for (auto const& entry: state) { network.appendList(4); - entry.endpoint.streamRLP(network, NodeIPEndpoint::InlineList); + entry.endpoint.streamRLP(network, NodeIPEndpoint::Inline); network << entry.id; count++; } diff --git a/libp2p/NodeTable.h b/libp2p/NodeTable.h index 4bb9a2178b8..76cd32097bd 100644 --- a/libp2p/NodeTable.h +++ b/libp2p/NodeTable.h @@ -299,8 +299,10 @@ struct InvalidRLP: public Exception {}; */ struct PingNode: RLPXDatagram { - PingNode(bi::udp::endpoint _ep): RLPXDatagram(_ep), source(UnspecifiedNodeIPEndpoint), destination(UnspecifiedNodeIPEndpoint) {} + /// Constructor used for sending PingNode. PingNode(NodeIPEndpoint _src, NodeIPEndpoint _dest): RLPXDatagram(_dest), source(_src), destination(_dest), ts(futureFromEpoch(std::chrono::seconds(60))) {} + /// Constructor used to create empty PingNode for parsing inbound packets. + PingNode(bi::udp::endpoint _ep): RLPXDatagram(_ep), source(UnspecifiedNodeIPEndpoint), destination(UnspecifiedNodeIPEndpoint) {} static const uint8_t type = 1; @@ -368,7 +370,7 @@ struct Neighbours: RLPXDatagram Neighbour(RLP const& _r): endpoint(_r) { node = h512(_r[3].toBytes()); } NodeIPEndpoint endpoint; NodeId node; - void streamRLP(RLPStream& _s) const { _s.appendList(4); endpoint.streamRLP(_s, NodeIPEndpoint::InlineList); _s << node; } + void streamRLP(RLPStream& _s) const { _s.appendList(4); endpoint.streamRLP(_s, NodeIPEndpoint::Inline); _s << node; } }; Neighbours(bi::udp::endpoint _ep): RLPXDatagram(_ep), ts(futureFromEpoch(std::chrono::seconds(30))) {} From e606ef9a59a545a095f43ac67b7dd5c9985c1837 Mon Sep 17 00:00:00 2001 From: subtly Date: Fri, 24 Apr 2015 11:12:03 +0100 Subject: [PATCH 08/22] check node discovery timestamps. --- libp2p/NodeTable.cpp | 10 ++++++++++ libp2p/NodeTable.h | 18 +++++++++--------- libp2p/UDP.h | 5 +++-- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/libp2p/NodeTable.cpp b/libp2p/NodeTable.cpp index 3370e502b26..7c30a9b0398 100644 --- a/libp2p/NodeTable.cpp +++ b/libp2p/NodeTable.cpp @@ -498,6 +498,11 @@ void NodeTable::onReceived(UDPSocketFace*, bi::udp::endpoint const& _from, bytes case FindNode::type: { FindNode in = FindNode::fromBytesConstRef(_from, rlpBytes); + if (RLPXDatagramFace::secondsSinceEpoch() - in.ts > 3) + { + clog(NodeTableTriviaSummary) << "Received expired FindNode from " << _from.address().to_string() << ":" << _from.port(); + return; + } vector> nearest = nearestNodeEntries(in.target); static unsigned const nlimit = (m_socketPointer->maxDatagramSize - 111) / 87; @@ -517,6 +522,11 @@ void NodeTable::onReceived(UDPSocketFace*, bi::udp::endpoint const& _from, bytes PingNode in = PingNode::fromBytesConstRef(_from, rlpBytes); if (in.version != dev::p2p::c_protocolVersion) return; + if (RLPXDatagramFace::secondsSinceEpoch() - in.ts > 3) + { + clog(NodeTableTriviaSummary) << "Received expired PingNode from " << _from.address().to_string() << ":" << _from.port(); + return; + } if (in.source.address.is_unspecified()) in.source.address = _from.address(); diff --git a/libp2p/NodeTable.h b/libp2p/NodeTable.h index 76cd32097bd..94bf8192487 100644 --- a/libp2p/NodeTable.h +++ b/libp2p/NodeTable.h @@ -300,7 +300,7 @@ struct InvalidRLP: public Exception {}; struct PingNode: RLPXDatagram { /// Constructor used for sending PingNode. - PingNode(NodeIPEndpoint _src, NodeIPEndpoint _dest): RLPXDatagram(_dest), source(_src), destination(_dest), ts(futureFromEpoch(std::chrono::seconds(60))) {} + PingNode(NodeIPEndpoint _src, NodeIPEndpoint _dest): RLPXDatagram(_dest), source(_src), destination(_dest), ts(secondsSinceEpoch()) {} /// Constructor used to create empty PingNode for parsing inbound packets. PingNode(bi::udp::endpoint _ep): RLPXDatagram(_ep), source(UnspecifiedNodeIPEndpoint), destination(UnspecifiedNodeIPEndpoint) {} @@ -309,7 +309,7 @@ struct PingNode: RLPXDatagram unsigned version = 0; NodeIPEndpoint source; NodeIPEndpoint destination; - unsigned ts; + uint32_t ts; void streamRLP(RLPStream& _s) const override; void interpretRLP(bytesConstRef _bytes) override; @@ -321,13 +321,13 @@ struct PingNode: RLPXDatagram struct Pong: RLPXDatagram { Pong(bi::udp::endpoint const& _ep): RLPXDatagram(_ep), destination(UnspecifiedNodeIPEndpoint) {} - Pong(NodeIPEndpoint const& _dest): RLPXDatagram((bi::udp::endpoint)_dest), destination(_dest), ts(futureFromEpoch(std::chrono::seconds(60))) {} + Pong(NodeIPEndpoint const& _dest): RLPXDatagram((bi::udp::endpoint)_dest), destination(_dest), ts(secondsSinceEpoch()) {} static const uint8_t type = 2; NodeIPEndpoint destination; h256 echo; ///< MCD of PingNode - unsigned ts; + uint32_t ts; void streamRLP(RLPStream& _s) const { _s.appendList(2); _s << echo << ts; } void interpretRLP(bytesConstRef _bytes) { RLP r(_bytes); echo = (h256)r[0]; ts = r[1].toInt(); } @@ -348,12 +348,12 @@ struct Pong: RLPXDatagram struct FindNode: RLPXDatagram { FindNode(bi::udp::endpoint _ep): RLPXDatagram(_ep) {} - FindNode(bi::udp::endpoint _ep, NodeId _target, std::chrono::seconds _ts = std::chrono::seconds(60)): RLPXDatagram(_ep), target(_target), ts(futureFromEpoch(_ts)) {} + FindNode(bi::udp::endpoint _ep, NodeId _target): RLPXDatagram(_ep), target(_target), ts(secondsSinceEpoch()) {} static const uint8_t type = 3; h512 target; - unsigned ts; + uint32_t ts; void streamRLP(RLPStream& _s) const { _s.appendList(2); _s << target << ts; } void interpretRLP(bytesConstRef _bytes) { RLP r(_bytes); target = r[0].toHash(); ts = r[1].toInt(); } @@ -373,8 +373,8 @@ struct Neighbours: RLPXDatagram void streamRLP(RLPStream& _s) const { _s.appendList(4); endpoint.streamRLP(_s, NodeIPEndpoint::Inline); _s << node; } }; - Neighbours(bi::udp::endpoint _ep): RLPXDatagram(_ep), ts(futureFromEpoch(std::chrono::seconds(30))) {} - Neighbours(bi::udp::endpoint _to, std::vector> const& _nearest, unsigned _offset = 0, unsigned _limit = 0): RLPXDatagram(_to), ts(futureFromEpoch(std::chrono::seconds(30))) + Neighbours(bi::udp::endpoint _ep): RLPXDatagram(_ep), ts(secondsSinceEpoch()) {} + Neighbours(bi::udp::endpoint _to, std::vector> const& _nearest, unsigned _offset = 0, unsigned _limit = 0): RLPXDatagram(_to), ts(secondsSinceEpoch()) { auto limit = _limit ? std::min(_nearest.size(), (size_t)(_offset + _limit)) : _nearest.size(); for (auto i = _offset; i < limit; i++) @@ -383,7 +383,7 @@ struct Neighbours: RLPXDatagram static const uint8_t type = 4; std::vector neighbours; - unsigned ts = 1; + uint32_t ts = 1; void streamRLP(RLPStream& _s) const { _s.appendList(2); _s.appendList(neighbours.size()); for (auto& n: neighbours) n.streamRLP(_s); _s << ts; } void interpretRLP(bytesConstRef _bytes) { RLP r(_bytes); for (auto n: r[0]) neighbours.push_back(Neighbour(n)); ts = r[1].toInt(); } diff --git a/libp2p/UDP.h b/libp2p/UDP.h index 374f986b01d..cdf241566dc 100644 --- a/libp2p/UDP.h +++ b/libp2p/UDP.h @@ -61,8 +61,9 @@ class UDPDatagram */ struct RLPXDatagramFace: public UDPDatagram { - static uint64_t futureFromEpoch(std::chrono::milliseconds _ms) { return std::chrono::duration_cast((std::chrono::system_clock::now() + _ms).time_since_epoch()).count(); } - static uint64_t futureFromEpoch(std::chrono::seconds _sec) { return std::chrono::duration_cast((std::chrono::system_clock::now() + _sec).time_since_epoch()).count(); } + static uint32_t futureFromEpoch(std::chrono::milliseconds _ms) { return std::chrono::duration_cast((std::chrono::system_clock::now() + _ms).time_since_epoch()).count(); } + static uint32_t futureFromEpoch(std::chrono::seconds _sec) { return std::chrono::duration_cast((std::chrono::system_clock::now() + _sec).time_since_epoch()).count(); } + static uint32_t secondsSinceEpoch() { return std::chrono::duration_cast((std::chrono::system_clock::now()).time_since_epoch()).count(); } static Public authenticate(bytesConstRef _sig, bytesConstRef _rlp); virtual uint8_t packetType() = 0; From 9c541f9694f8be41393c8de5ff86960e04e37697 Mon Sep 17 00:00:00 2001 From: subtly Date: Fri, 24 Apr 2015 11:54:51 +0100 Subject: [PATCH 09/22] simple endpoint update --- libp2p/NodeTable.cpp | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/libp2p/NodeTable.cpp b/libp2p/NodeTable.cpp index 7c30a9b0398..c1d3447d5f1 100644 --- a/libp2p/NodeTable.cpp +++ b/libp2p/NodeTable.cpp @@ -313,12 +313,12 @@ void NodeTable::noteActiveNode(Public const& _pubk, bi::udp::endpoint const& _en return; shared_ptr node = nodeEntry(_pubk); - if (!!node && !node->pending && !!node->endpoint) + node->endpoint.address = _endpoint.address(); + node->endpoint.udpPort = _endpoint.port(); + if (!!node && !node->pending) { // todo: drop in favor of ping/pong packets clog(NodeTableConnect) << "Noting active node:" << _pubk.abridged() << _endpoint.address().to_string() << ":" << _endpoint.port(); - node->endpoint.address = _endpoint.address(); - node->endpoint.udpPort = _endpoint.port(); shared_ptr contested; { @@ -456,15 +456,10 @@ void NodeTable::onReceived(UDPSocketFace*, bi::udp::endpoint const& _from, bytes return; // unsolicited pong; don't note node as active } - // update our endpoint address and UDP port (with caution and iff appropriate) - if (false && !m_node.endpoint) - { - if (in.destination.address != m_node.endpoint.address) - m_node.endpoint.address = in.destination.address; - - if (in.destination.udpPort != m_node.endpoint.udpPort) - m_node.endpoint.udpPort = in.destination.udpPort; - } + // update our endpoint address and UDP port + if ((!m_node.endpoint || !m_node.endpoint.isAllowed()) && in.destination.address != m_node.endpoint.address && isPublicAddress(in.destination.address)) + m_node.endpoint.address = in.destination.address; + m_node.endpoint.udpPort = in.destination.udpPort; clog(NodeTableConnect) << "PONG from " << nodeid.abridged() << _from; break; @@ -528,10 +523,10 @@ void NodeTable::onReceived(UDPSocketFace*, bi::udp::endpoint const& _from, bytes return; } - if (in.source.address.is_unspecified()) - in.source.address = _from.address(); + in.source.address = _from.address(); + in.source.udpPort = _from.port(); addNode(Node(nodeid, in.source)); - Pong p(NodeIPEndpoint(_from.address(), _from.port(), in.source.tcpPort)); + Pong p(in.source); p.echo = sha3(rlpBytes); p.sign(m_secret); m_socketPointer->send(p); From 12fbcc8b5f44a6ca5c3e59b5ac2b4ed09739d439 Mon Sep 17 00:00:00 2001 From: subtly Date: Fri, 24 Apr 2015 12:12:27 +0100 Subject: [PATCH 10/22] cleanup --- libp2p/NodeTable.cpp | 50 ++++++++++++++++++++------------------------ 1 file changed, 23 insertions(+), 27 deletions(-) diff --git a/libp2p/NodeTable.cpp b/libp2p/NodeTable.cpp index 4b1590a5345..2f69d606d32 100644 --- a/libp2p/NodeTable.cpp +++ b/libp2p/NodeTable.cpp @@ -218,24 +218,22 @@ vector> NodeTable::nearestNodeEntries(NodeId _target) Guard l(x_state); for (auto n: m_state[head].nodes) if (auto p = n.lock()) - if (!!p->endpoint && p->endpoint.isAllowed()) + { + if (count < s_bucketSize) + found[distance(_target, p->id)].push_back(p); + else + break; + } + + if (count < s_bucketSize && tail) + for (auto n: m_state[tail].nodes) + if (auto p = n.lock()) { if (count < s_bucketSize) found[distance(_target, p->id)].push_back(p); else break; } - - if (count < s_bucketSize && tail) - for (auto n: m_state[tail].nodes) - if (auto p = n.lock()) - if (!!p->endpoint && p->endpoint.isAllowed()) - { - if (count < s_bucketSize) - found[distance(_target, p->id)].push_back(p); - else - break; - } head++; if (tail) @@ -247,13 +245,12 @@ vector> NodeTable::nearestNodeEntries(NodeId _target) Guard l(x_state); for (auto n: m_state[head].nodes) if (auto p = n.lock()) - if (!!p->endpoint && p->endpoint.isAllowed()) - { - if (count < s_bucketSize) - found[distance(_target, p->id)].push_back(p); - else - break; - } + { + if (count < s_bucketSize) + found[distance(_target, p->id)].push_back(p); + else + break; + } head++; } else @@ -262,20 +259,19 @@ vector> NodeTable::nearestNodeEntries(NodeId _target) Guard l(x_state); for (auto n: m_state[tail].nodes) if (auto p = n.lock()) - if (!!p->endpoint && p->endpoint.isAllowed()) - { - if (count < s_bucketSize) - found[distance(_target, p->id)].push_back(p); - else - break; - } + { + if (count < s_bucketSize) + found[distance(_target, p->id)].push_back(p); + else + break; + } tail--; } vector> ret; for (auto& nodes: found) for (auto n: nodes.second) - if (ret.size() < s_bucketSize && n->endpoint.isAllowed()) + if (ret.size() < s_bucketSize && !!n->endpoint && n->endpoint.isAllowed()) ret.push_back(n); return move(ret); } From eb946bb1edbbefdf96699e6fdb691bd555c71975 Mon Sep 17 00:00:00 2001 From: subtly Date: Fri, 24 Apr 2015 12:23:27 +0100 Subject: [PATCH 11/22] cleanup --- libp2p/NodeTable.cpp | 8 +++----- libp2p/NodeTable.h | 6 +++--- libp2p/UDP.h | 2 -- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/libp2p/NodeTable.cpp b/libp2p/NodeTable.cpp index 2f69d606d32..ad006e83710 100644 --- a/libp2p/NodeTable.cpp +++ b/libp2p/NodeTable.cpp @@ -414,8 +414,6 @@ void NodeTable::onReceived(UDPSocketFace*, bi::udp::endpoint const& _from, bytes { Pong in = Pong::fromBytesConstRef(_from, rlpBytes); - // TODO: check echo! (pending pings) - // whenever a pong is received, check if it's in m_evictions Guard le(x_evictions); bool evictionEntry = false; @@ -451,7 +449,7 @@ void NodeTable::onReceived(UDPSocketFace*, bi::udp::endpoint const& _from, bytes } // update our endpoint address and UDP port - if ((!m_node.endpoint || !m_node.endpoint.isAllowed()) && in.destination.address != m_node.endpoint.address && isPublicAddress(in.destination.address)) + if ((!m_node.endpoint || !m_node.endpoint.isAllowed()) && isPublicAddress(in.destination.address)) m_node.endpoint.address = in.destination.address; m_node.endpoint.udpPort = in.destination.udpPort; @@ -516,7 +514,7 @@ void NodeTable::onReceived(UDPSocketFace*, bi::udp::endpoint const& _from, bytes clog(NodeTableTriviaSummary) << "Received expired PingNode from " << _from.address().to_string() << ":" << _from.port(); return; } - + in.source.address = _from.address(); in.source.udpPort = _from.port(); addNode(Node(nodeid, in.source)); @@ -610,7 +608,7 @@ void PingNode::interpretRLP(bytesConstRef _bytes) version = dev::p2p::c_protocolVersion; source.interpretRLP(r[1]); destination.interpretRLP(r[2]); - ts = r[3].toInt(RLP::Strict); + ts = r[3].toInt(RLP::Strict); } else version = 0; diff --git a/libp2p/NodeTable.h b/libp2p/NodeTable.h index 94bf8192487..8c1d10a382b 100644 --- a/libp2p/NodeTable.h +++ b/libp2p/NodeTable.h @@ -330,7 +330,7 @@ struct Pong: RLPXDatagram uint32_t ts; void streamRLP(RLPStream& _s) const { _s.appendList(2); _s << echo << ts; } - void interpretRLP(bytesConstRef _bytes) { RLP r(_bytes); echo = (h256)r[0]; ts = r[1].toInt(); } + void interpretRLP(bytesConstRef _bytes) { RLP r(_bytes); echo = (h256)r[0]; ts = r[1].toInt(); } }; /** @@ -356,7 +356,7 @@ struct FindNode: RLPXDatagram uint32_t ts; void streamRLP(RLPStream& _s) const { _s.appendList(2); _s << target << ts; } - void interpretRLP(bytesConstRef _bytes) { RLP r(_bytes); target = r[0].toHash(); ts = r[1].toInt(); } + void interpretRLP(bytesConstRef _bytes) { RLP r(_bytes); target = r[0].toHash(); ts = r[1].toInt(); } }; /** @@ -386,7 +386,7 @@ struct Neighbours: RLPXDatagram uint32_t ts = 1; void streamRLP(RLPStream& _s) const { _s.appendList(2); _s.appendList(neighbours.size()); for (auto& n: neighbours) n.streamRLP(_s); _s << ts; } - void interpretRLP(bytesConstRef _bytes) { RLP r(_bytes); for (auto n: r[0]) neighbours.push_back(Neighbour(n)); ts = r[1].toInt(); } + void interpretRLP(bytesConstRef _bytes) { RLP r(_bytes); for (auto n: r[0]) neighbours.push_back(Neighbour(n)); ts = r[1].toInt(); } }; struct NodeTableWarn: public LogChannel { static const char* name(); static const int verbosity = 0; }; diff --git a/libp2p/UDP.h b/libp2p/UDP.h index cdf241566dc..ee9875cf758 100644 --- a/libp2p/UDP.h +++ b/libp2p/UDP.h @@ -61,8 +61,6 @@ class UDPDatagram */ struct RLPXDatagramFace: public UDPDatagram { - static uint32_t futureFromEpoch(std::chrono::milliseconds _ms) { return std::chrono::duration_cast((std::chrono::system_clock::now() + _ms).time_since_epoch()).count(); } - static uint32_t futureFromEpoch(std::chrono::seconds _sec) { return std::chrono::duration_cast((std::chrono::system_clock::now() + _sec).time_since_epoch()).count(); } static uint32_t secondsSinceEpoch() { return std::chrono::duration_cast((std::chrono::system_clock::now()).time_since_epoch()).count(); } static Public authenticate(bytesConstRef _sig, bytesConstRef _rlp); From 00cbab85aee3555e0d97a91f1bf6e82e311adfc3 Mon Sep 17 00:00:00 2001 From: subtly Date: Fri, 24 Apr 2015 18:18:14 +0100 Subject: [PATCH 12/22] fix for insensitive windows compiler which reserves 'inline'. default initial values for packet timestamps. --- libp2p/Common.h | 6 +++--- libp2p/Host.cpp | 4 ++-- libp2p/NodeTable.h | 10 +++++----- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/libp2p/Common.h b/libp2p/Common.h index 0b572473760..f4b5d660395 100644 --- a/libp2p/Common.h +++ b/libp2p/Common.h @@ -165,8 +165,8 @@ struct NodeIPEndpoint { enum RLPAppend { - List, - Inline + StreamList, + StreamInline }; /// Setting true causes isAllowed to return true for all addresses. (Used by test fixtures) @@ -186,7 +186,7 @@ struct NodeIPEndpoint bool isAllowed() const { return NodeIPEndpoint::test_allowLocal ? !address.is_unspecified() : isPublicAddress(address); } - void streamRLP(RLPStream& _s, RLPAppend _inline = List) const { if (_inline == List) _s.appendList(3); if (address.is_v4()) _s << address.to_v4().to_bytes(); else if (address.is_v6()) _s << address.to_v6().to_bytes(); else _s << ""; _s << udpPort << tcpPort; } + void streamRLP(RLPStream& _s, RLPAppend _inline = StreamList) const { if (_inline == StreamList) _s.appendList(3); if (address.is_v4()) _s << address.to_v4().to_bytes(); else if (address.is_v6()) _s << address.to_v6().to_bytes(); else _s << ""; _s << udpPort << tcpPort; } void interpretRLP(RLP const& _r) { if (_r[0].size() == 0) address = bi::address(); else if (_r[0].size() == 4) address = bi::address_v4(_r[0].toArray()); else address = bi::address_v6(_r[0].toArray()); udpPort = _r[1].toInt(); tcpPort = _r[2].toInt(); } }; diff --git a/libp2p/Host.cpp b/libp2p/Host.cpp index 11e4111f8ca..d70d22df157 100644 --- a/libp2p/Host.cpp +++ b/libp2p/Host.cpp @@ -686,7 +686,7 @@ bytes Host::saveNetwork() const if (chrono::system_clock::now() - p.m_lastConnected < chrono::seconds(3600 * 48) && !!p.endpoint && p.id != id() && (p.required || p.endpoint.isAllowed())) { network.appendList(11); - p.endpoint.streamRLP(network, NodeIPEndpoint::Inline); + p.endpoint.streamRLP(network, NodeIPEndpoint::StreamInline); network << p.id << p.required << chrono::duration_cast(p.m_lastConnected.time_since_epoch()).count() << chrono::duration_cast(p.m_lastAttempted.time_since_epoch()).count() @@ -702,7 +702,7 @@ bytes Host::saveNetwork() const for (auto const& entry: state) { network.appendList(4); - entry.endpoint.streamRLP(network, NodeIPEndpoint::Inline); + entry.endpoint.streamRLP(network, NodeIPEndpoint::StreamInline); network << entry.id; count++; } diff --git a/libp2p/NodeTable.h b/libp2p/NodeTable.h index 8c1d10a382b..819ea3ba5c5 100644 --- a/libp2p/NodeTable.h +++ b/libp2p/NodeTable.h @@ -309,7 +309,7 @@ struct PingNode: RLPXDatagram unsigned version = 0; NodeIPEndpoint source; NodeIPEndpoint destination; - uint32_t ts; + uint32_t ts = 0; void streamRLP(RLPStream& _s) const override; void interpretRLP(bytesConstRef _bytes) override; @@ -327,7 +327,7 @@ struct Pong: RLPXDatagram NodeIPEndpoint destination; h256 echo; ///< MCD of PingNode - uint32_t ts; + uint32_t ts = 0; void streamRLP(RLPStream& _s) const { _s.appendList(2); _s << echo << ts; } void interpretRLP(bytesConstRef _bytes) { RLP r(_bytes); echo = (h256)r[0]; ts = r[1].toInt(); } @@ -353,7 +353,7 @@ struct FindNode: RLPXDatagram static const uint8_t type = 3; h512 target; - uint32_t ts; + uint32_t ts = 0; void streamRLP(RLPStream& _s) const { _s.appendList(2); _s << target << ts; } void interpretRLP(bytesConstRef _bytes) { RLP r(_bytes); target = r[0].toHash(); ts = r[1].toInt(); } @@ -370,7 +370,7 @@ struct Neighbours: RLPXDatagram Neighbour(RLP const& _r): endpoint(_r) { node = h512(_r[3].toBytes()); } NodeIPEndpoint endpoint; NodeId node; - void streamRLP(RLPStream& _s) const { _s.appendList(4); endpoint.streamRLP(_s, NodeIPEndpoint::Inline); _s << node; } + void streamRLP(RLPStream& _s) const { _s.appendList(4); endpoint.streamRLP(_s, NodeIPEndpoint::StreamInline); _s << node; } }; Neighbours(bi::udp::endpoint _ep): RLPXDatagram(_ep), ts(secondsSinceEpoch()) {} @@ -383,7 +383,7 @@ struct Neighbours: RLPXDatagram static const uint8_t type = 4; std::vector neighbours; - uint32_t ts = 1; + uint32_t ts = 0; void streamRLP(RLPStream& _s) const { _s.appendList(2); _s.appendList(neighbours.size()); for (auto& n: neighbours) n.streamRLP(_s); _s << ts; } void interpretRLP(bytesConstRef _bytes) { RLP r(_bytes); for (auto n: r[0]) neighbours.push_back(Neighbour(n)); ts = r[1].toInt(); } From 52213f258e2724c3eef4e27c85d04619e1600e31 Mon Sep 17 00:00:00 2001 From: subtly Date: Mon, 27 Apr 2015 14:17:30 +0100 Subject: [PATCH 13/22] encapsulate IP endpoint address as byte string --- libp2p/Common.cpp | 25 +++++++++++++++++++++++++ libp2p/Common.h | 4 ++-- libp2p/NodeTable.cpp | 2 +- test/libp2p/net.cpp | 4 ++-- test/libp2p/peer.cpp | 2 +- 5 files changed, 31 insertions(+), 6 deletions(-) diff --git a/libp2p/Common.cpp b/libp2p/Common.cpp index a2e1398b539..192dc9c4cd3 100644 --- a/libp2p/Common.cpp +++ b/libp2p/Common.cpp @@ -144,6 +144,31 @@ std::string p2p::reasonOf(DisconnectReason _r) } } +void NodeIPEndpoint::streamRLP(RLPStream& _s, RLPAppend _inline) const +{ + if (_inline == StreamList) + _s.appendList(3); + if (address.is_v4()) + _s << bytesConstRef(&address.to_v4().to_bytes()[0], 4); + else if (address.is_v6()) + _s << bytesConstRef(&address.to_v6().to_bytes()[0], 16); + else + _s << bytes(); + _s << udpPort << tcpPort; +} + +void NodeIPEndpoint::interpretRLP(RLP const& _r) +{ + if (_r[0].size() == 4) + address = bi::address_v4(*(bi::address_v4::bytes_type*)_r[0].toBytes().data()); + else if (_r[0].size() == 16) + address = bi::address_v6(*(bi::address_v6::bytes_type*)_r[0].toBytes().data()); + else + address = bi::address(); + udpPort = _r[1].toInt(); + tcpPort = _r[2].toInt(); +} + namespace dev { std::ostream& operator<<(std::ostream& _out, dev::p2p::NodeIPEndpoint const& _ep) diff --git a/libp2p/Common.h b/libp2p/Common.h index f4b5d660395..0a6867599ff 100644 --- a/libp2p/Common.h +++ b/libp2p/Common.h @@ -186,8 +186,8 @@ struct NodeIPEndpoint bool isAllowed() const { return NodeIPEndpoint::test_allowLocal ? !address.is_unspecified() : isPublicAddress(address); } - void streamRLP(RLPStream& _s, RLPAppend _inline = StreamList) const { if (_inline == StreamList) _s.appendList(3); if (address.is_v4()) _s << address.to_v4().to_bytes(); else if (address.is_v6()) _s << address.to_v6().to_bytes(); else _s << ""; _s << udpPort << tcpPort; } - void interpretRLP(RLP const& _r) { if (_r[0].size() == 0) address = bi::address(); else if (_r[0].size() == 4) address = bi::address_v4(_r[0].toArray()); else address = bi::address_v6(_r[0].toArray()); udpPort = _r[1].toInt(); tcpPort = _r[2].toInt(); } + void streamRLP(RLPStream& _s, RLPAppend _inline = StreamList) const; + void interpretRLP(RLP const& _r); }; struct Node diff --git a/libp2p/NodeTable.cpp b/libp2p/NodeTable.cpp index ad006e83710..154032b6d76 100644 --- a/libp2p/NodeTable.cpp +++ b/libp2p/NodeTable.cpp @@ -492,7 +492,7 @@ void NodeTable::onReceived(UDPSocketFace*, bi::udp::endpoint const& _from, bytes } vector> nearest = nearestNodeEntries(in.target); - static unsigned const nlimit = (m_socketPointer->maxDatagramSize - 111) / 87; + static unsigned const nlimit = (m_socketPointer->maxDatagramSize - 109) / 90; for (unsigned offset = 0; offset < nearest.size(); offset += nlimit) { Neighbours out(_from, nearest, offset, nlimit); diff --git a/test/libp2p/net.cpp b/test/libp2p/net.cpp index f2808364e61..a95e685c7d6 100644 --- a/test/libp2p/net.cpp +++ b/test/libp2p/net.cpp @@ -235,8 +235,8 @@ BOOST_AUTO_TEST_CASE(neighboursPacketLength) std::vector> testNodes(TestNodeTable::createTestNodes(16)); bi::udp::endpoint to(boost::asio::ip::address::from_string("127.0.0.1"), 30000); - // hash(32), signature(65), overhead: packet(2), type(1), nodeList(2), ts(9), - static unsigned const nlimit = (1280 - 111) / 87; + // hash(32), signature(65), overhead: packetSz(3), type(1), nodeListSz(3), ts(5), + static unsigned const nlimit = (1280 - 109) / 90; // neighbour: 2 + 65 + 3 + 3 + 17 for (unsigned offset = 0; offset < testNodes.size(); offset += nlimit) { Neighbours out(to); diff --git a/test/libp2p/peer.cpp b/test/libp2p/peer.cpp index 7532cda0ecd..7ba698edd9f 100644 --- a/test/libp2p/peer.cpp +++ b/test/libp2p/peer.cpp @@ -114,7 +114,7 @@ BOOST_AUTO_TEST_CASE(saveNodes) for (auto i: r[2]) { BOOST_REQUIRE(i.itemCount() == 4 || i.itemCount() == 11); - BOOST_REQUIRE(i[0].itemCount() == 4 || i[0].itemCount() == 16); + BOOST_REQUIRE(i[0].size() == 4 || i[0].size() == 16); } } From 52a219c4ce3c42dbbb8088fdaf84d45343807dda Mon Sep 17 00:00:00 2001 From: subtly Date: Tue, 28 Apr 2015 13:57:35 +0100 Subject: [PATCH 14/22] revert code from timestamps back to cache timers --- libp2p/NodeTable.cpp | 4 ++-- libp2p/NodeTable.h | 9 +++++---- libp2p/UDP.h | 1 + 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/libp2p/NodeTable.cpp b/libp2p/NodeTable.cpp index 154032b6d76..24380a10edc 100644 --- a/libp2p/NodeTable.cpp +++ b/libp2p/NodeTable.cpp @@ -485,7 +485,7 @@ void NodeTable::onReceived(UDPSocketFace*, bi::udp::endpoint const& _from, bytes case FindNode::type: { FindNode in = FindNode::fromBytesConstRef(_from, rlpBytes); - if (RLPXDatagramFace::secondsSinceEpoch() - in.ts > 3) + if (RLPXDatagramFace::secondsSinceEpoch() > in.ts) { clog(NodeTableTriviaSummary) << "Received expired FindNode from " << _from.address().to_string() << ":" << _from.port(); return; @@ -509,7 +509,7 @@ void NodeTable::onReceived(UDPSocketFace*, bi::udp::endpoint const& _from, bytes PingNode in = PingNode::fromBytesConstRef(_from, rlpBytes); if (in.version != dev::p2p::c_protocolVersion) return; - if (RLPXDatagramFace::secondsSinceEpoch() - in.ts > 3) + if (RLPXDatagramFace::secondsSinceEpoch() > in.ts) { clog(NodeTableTriviaSummary) << "Received expired PingNode from " << _from.address().to_string() << ":" << _from.port(); return; diff --git a/libp2p/NodeTable.h b/libp2p/NodeTable.h index 819ea3ba5c5..e4eddb9573c 100644 --- a/libp2p/NodeTable.h +++ b/libp2p/NodeTable.h @@ -300,7 +300,8 @@ struct InvalidRLP: public Exception {}; struct PingNode: RLPXDatagram { /// Constructor used for sending PingNode. - PingNode(NodeIPEndpoint _src, NodeIPEndpoint _dest): RLPXDatagram(_dest), source(_src), destination(_dest), ts(secondsSinceEpoch()) {} + PingNode(NodeIPEndpoint _src, NodeIPEndpoint _dest): RLPXDatagram(_dest), source(_src), destination(_dest), ts(futureFromEpoch(std::chrono::seconds(60))) {} + /// Constructor used to create empty PingNode for parsing inbound packets. PingNode(bi::udp::endpoint _ep): RLPXDatagram(_ep), source(UnspecifiedNodeIPEndpoint), destination(UnspecifiedNodeIPEndpoint) {} @@ -321,7 +322,7 @@ struct PingNode: RLPXDatagram struct Pong: RLPXDatagram { Pong(bi::udp::endpoint const& _ep): RLPXDatagram(_ep), destination(UnspecifiedNodeIPEndpoint) {} - Pong(NodeIPEndpoint const& _dest): RLPXDatagram((bi::udp::endpoint)_dest), destination(_dest), ts(secondsSinceEpoch()) {} + Pong(NodeIPEndpoint const& _dest): RLPXDatagram((bi::udp::endpoint)_dest), destination(_dest), ts(futureFromEpoch(std::chrono::seconds(60))) {} static const uint8_t type = 2; @@ -348,7 +349,7 @@ struct Pong: RLPXDatagram struct FindNode: RLPXDatagram { FindNode(bi::udp::endpoint _ep): RLPXDatagram(_ep) {} - FindNode(bi::udp::endpoint _ep, NodeId _target): RLPXDatagram(_ep), target(_target), ts(secondsSinceEpoch()) {} + FindNode(bi::udp::endpoint _ep, NodeId _target): RLPXDatagram(_ep), target(_target), ts(futureFromEpoch(std::chrono::seconds(60))) {} static const uint8_t type = 3; @@ -374,7 +375,7 @@ struct Neighbours: RLPXDatagram }; Neighbours(bi::udp::endpoint _ep): RLPXDatagram(_ep), ts(secondsSinceEpoch()) {} - Neighbours(bi::udp::endpoint _to, std::vector> const& _nearest, unsigned _offset = 0, unsigned _limit = 0): RLPXDatagram(_to), ts(secondsSinceEpoch()) + Neighbours(bi::udp::endpoint _to, std::vector> const& _nearest, unsigned _offset = 0, unsigned _limit = 0): RLPXDatagram(_to), ts(futureFromEpoch(std::chrono::seconds(60))) { auto limit = _limit ? std::min(_nearest.size(), (size_t)(_offset + _limit)) : _nearest.size(); for (auto i = _offset; i < limit; i++) diff --git a/libp2p/UDP.h b/libp2p/UDP.h index ee9875cf758..97136ee4399 100644 --- a/libp2p/UDP.h +++ b/libp2p/UDP.h @@ -61,6 +61,7 @@ class UDPDatagram */ struct RLPXDatagramFace: public UDPDatagram { + static uint32_t futureFromEpoch(std::chrono::seconds _sec) { return std::chrono::duration_cast((std::chrono::system_clock::now() + _sec).time_since_epoch()).count(); } static uint32_t secondsSinceEpoch() { return std::chrono::duration_cast((std::chrono::system_clock::now()).time_since_epoch()).count(); } static Public authenticate(bytesConstRef _sig, bytesConstRef _rlp); From 30acd1343ade4d887927c72d4372e23d192ca197 Mon Sep 17 00:00:00 2001 From: subtly Date: Tue, 28 Apr 2015 14:18:40 +0100 Subject: [PATCH 15/22] Windows keyword error for _inline. --- libp2p/Common.cpp | 4 ++-- libp2p/Common.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libp2p/Common.cpp b/libp2p/Common.cpp index 192dc9c4cd3..e2ae302550a 100644 --- a/libp2p/Common.cpp +++ b/libp2p/Common.cpp @@ -144,9 +144,9 @@ std::string p2p::reasonOf(DisconnectReason _r) } } -void NodeIPEndpoint::streamRLP(RLPStream& _s, RLPAppend _inline) const +void NodeIPEndpoint::streamRLP(RLPStream& _s, RLPAppend _append) const { - if (_inline == StreamList) + if (_append == StreamList) _s.appendList(3); if (address.is_v4()) _s << bytesConstRef(&address.to_v4().to_bytes()[0], 4); diff --git a/libp2p/Common.h b/libp2p/Common.h index 0a6867599ff..04faf0f24fd 100644 --- a/libp2p/Common.h +++ b/libp2p/Common.h @@ -186,7 +186,7 @@ struct NodeIPEndpoint bool isAllowed() const { return NodeIPEndpoint::test_allowLocal ? !address.is_unspecified() : isPublicAddress(address); } - void streamRLP(RLPStream& _s, RLPAppend _inline = StreamList) const; + void streamRLP(RLPStream& _s, RLPAppend _append = StreamList) const; void interpretRLP(RLP const& _r); }; From c84d5f30bb4b1808ae92afb1ce3a4e56c10df728 Mon Sep 17 00:00:00 2001 From: subtly Date: Thu, 30 Apr 2015 12:30:34 +0100 Subject: [PATCH 16/22] update pong encoding --- libp2p/NodeTable.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libp2p/NodeTable.h b/libp2p/NodeTable.h index e4eddb9573c..ea0f31d503f 100644 --- a/libp2p/NodeTable.h +++ b/libp2p/NodeTable.h @@ -330,8 +330,8 @@ struct Pong: RLPXDatagram h256 echo; ///< MCD of PingNode uint32_t ts = 0; - void streamRLP(RLPStream& _s) const { _s.appendList(2); _s << echo << ts; } - void interpretRLP(bytesConstRef _bytes) { RLP r(_bytes); echo = (h256)r[0]; ts = r[1].toInt(); } + void streamRLP(RLPStream& _s) const { _s.appendList(3); destination.streamRLP(_s); _s << echo << ts; } + void interpretRLP(bytesConstRef _bytes) { RLP r(_bytes); destination.interpretRLP(r[0]); echo = (h256)r[1]; ts = r[2].toInt(); } }; /** From f176a1ce137756dcc5bdea34abd311a407194a1d Mon Sep 17 00:00:00 2001 From: subtly Date: Thu, 30 Apr 2015 16:16:38 +0100 Subject: [PATCH 17/22] add missing const --- libp2p/Common.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libp2p/Common.h b/libp2p/Common.h index 04faf0f24fd..15ee981bc95 100644 --- a/libp2p/Common.h +++ b/libp2p/Common.h @@ -192,7 +192,7 @@ struct NodeIPEndpoint struct Node { - Node(Public _pubk, NodeIPEndpoint _ip, bool _required = false): id(_pubk), endpoint(_ip), required(_required) {} + Node(Public _pubk, NodeIPEndpoint const& _ip, bool _required = false): id(_pubk), endpoint(_ip), required(_required) {} virtual NodeId const& address() const { return id; } virtual Public const& publicKey() const { return id; } From 3ebaffa2c12b9cf8604f36c3637877d1accf6cec Mon Sep 17 00:00:00 2001 From: subtly Date: Thu, 30 Apr 2015 17:02:14 +0100 Subject: [PATCH 18/22] revert assumption that node exists in nodetable when noteactivenode is called --- libp2p/NodeTable.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libp2p/NodeTable.cpp b/libp2p/NodeTable.cpp index 24380a10edc..845d892a1b5 100644 --- a/libp2p/NodeTable.cpp +++ b/libp2p/NodeTable.cpp @@ -309,11 +309,12 @@ void NodeTable::noteActiveNode(Public const& _pubk, bi::udp::endpoint const& _en return; shared_ptr node = nodeEntry(_pubk); - node->endpoint.address = _endpoint.address(); - node->endpoint.udpPort = _endpoint.port(); if (!!node && !node->pending) { clog(NodeTableConnect) << "Noting active node:" << _pubk << _endpoint.address().to_string() << ":" << _endpoint.port(); + node->endpoint.address = _endpoint.address(); + node->endpoint.udpPort = _endpoint.port(); + shared_ptr contested; { Guard l(x_state); From d46bd2b5a300fb6d260300393e260785c81a0a28 Mon Sep 17 00:00:00 2001 From: subtly Date: Wed, 6 May 2015 23:34:10 +0200 Subject: [PATCH 19/22] Missed in merge. --- libp2p/NodeTable.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libp2p/NodeTable.cpp b/libp2p/NodeTable.cpp index 7f98edb838c..9d1ba2ba4cc 100644 --- a/libp2p/NodeTable.cpp +++ b/libp2p/NodeTable.cpp @@ -75,7 +75,7 @@ void NodeTable::processEvents() m_nodeEventHandler->processEvents(); } -shared_ptr NodeTable::addNode(Node const& _node, NodeRelation _relation = Unknown) +shared_ptr NodeTable::addNode(Node const& _node, NodeRelation _relation) { if (_relation == Known) { From 03db500f330b511e8ea191ca5a3df8efe21153df Mon Sep 17 00:00:00 2001 From: subtly Date: Wed, 6 May 2015 23:38:32 +0200 Subject: [PATCH 20/22] Merge stash into discovery. --- libp2p/Common.cpp | 1 + libp2p/Host.cpp | 70 +++++++++++++++++++++++++++++++------------- libp2p/NodeTable.cpp | 39 ++++++++++++++++++++++-- libp2p/NodeTable.h | 33 +++++++++++++-------- 4 files changed, 107 insertions(+), 36 deletions(-) diff --git a/libp2p/Common.cpp b/libp2p/Common.cpp index e2ae302550a..ed84f014cc2 100644 --- a/libp2p/Common.cpp +++ b/libp2p/Common.cpp @@ -26,6 +26,7 @@ using namespace dev::p2p; const unsigned dev::p2p::c_protocolVersion = 4; const unsigned dev::p2p::c_defaultIPPort = 30303; +static_assert(dev::p2p::c_protocolVersion == 4, "Replace v3 compatbility with v4 compatibility before updating network version."); const dev::p2p::NodeIPEndpoint dev::p2p::UnspecifiedNodeIPEndpoint = NodeIPEndpoint(bi::address(), 0, 0); const dev::p2p::Node dev::p2p::UnspecifiedNode = dev::p2p::Node(NodeId(), UnspecifiedNodeIPEndpoint); diff --git a/libp2p/Host.cpp b/libp2p/Host.cpp index ca05413eddb..f781fc1042e 100644 --- a/libp2p/Host.cpp +++ b/libp2p/Host.cpp @@ -746,7 +746,8 @@ void Host::restoreNetwork(bytesConstRef _b) RecursiveGuard l(x_sessions); RLP r(_b); - if (r.itemCount() > 0 && r[0].isInt() && r[0].toInt() == dev::p2p::c_protocolVersion) + unsigned fileVersion = r[0].toInt(); + if (r.itemCount() > 0 && r[0].isInt() && fileVersion >= dev::p2p::c_protocolVersion - 1) { // r[0] = version // r[1] = key @@ -758,26 +759,53 @@ void Host::restoreNetwork(bytesConstRef _b) if (i[0].itemCount() != 4) continue; - Node n((NodeId)i[3], NodeIPEndpoint(i)); - if (i.itemCount() == 4 && n.endpoint.isAllowed()) - m_nodeTable->addNode(n); - else if (i.itemCount() == 11) + if (i.itemCount() == 4 || i.itemCount() == 11) { - n.required = i[4].toInt(); - if (!n.endpoint.isAllowed() && !n.required) - continue; - shared_ptr p = make_shared(n); - p->m_lastConnected = chrono::system_clock::time_point(chrono::seconds(i[5].toInt())); - p->m_lastAttempted = chrono::system_clock::time_point(chrono::seconds(i[6].toInt())); - p->m_failedAttempts = i[7].toInt(); - p->m_lastDisconnect = (DisconnectReason)i[8].toInt(); - p->m_score = (int)i[9].toInt(); - p->m_rating = (int)i[10].toInt(); - m_peers[p->id] = p; - if (p->required) - requirePeer(p->id, n.endpoint); - else - m_nodeTable->addNode(*p.get(), NodeTable::NodeRelation::Known); + Node n((NodeId)i[3], NodeIPEndpoint(i)); + if (i.itemCount() == 4 && n.endpoint.isAllowed()) + m_nodeTable->addNode(n); + else if (i.itemCount() == 11) + { + n.required = i[4].toInt(); + if (!n.endpoint.isAllowed() && !n.required) + continue; + shared_ptr p = make_shared(n); + p->m_lastConnected = chrono::system_clock::time_point(chrono::seconds(i[5].toInt())); + p->m_lastAttempted = chrono::system_clock::time_point(chrono::seconds(i[6].toInt())); + p->m_failedAttempts = i[7].toInt(); + p->m_lastDisconnect = (DisconnectReason)i[8].toInt(); + p->m_score = (int)i[9].toInt(); + p->m_rating = (int)i[10].toInt(); + m_peers[p->id] = p; + if (p->required) + requirePeer(p->id, n.endpoint); + else + m_nodeTable->addNode(*p.get(), NodeTable::NodeRelation::Known); + } + } + else if (i.itemCount() == 3 || i.itemCount() == 10) + { + Node n((NodeId)i[2], NodeIPEndpoint(bi::address_v4(i[0].toArray()), i[1].toInt(), i[1].toInt())); + if (i.itemCount() == 3 && n.endpoint.isAllowed()) + m_nodeTable->addNode(n); + else if (i.itemCount() == 10) + { + n.required = i[3].toInt(); + if (!n.endpoint.isAllowed() && !n.required) + continue; + shared_ptr p = make_shared(n); + p->m_lastConnected = chrono::system_clock::time_point(chrono::seconds(i[4].toInt())); + p->m_lastAttempted = chrono::system_clock::time_point(chrono::seconds(i[5].toInt())); + p->m_failedAttempts = i[6].toInt(); + p->m_lastDisconnect = (DisconnectReason)i[7].toInt(); + p->m_score = (int)i[8].toInt(); + p->m_rating = (int)i[9].toInt(); + m_peers[p->id] = p; + if (p->required) + requirePeer(p->id, n.endpoint); + else + m_nodeTable->addNode(*p.get(), NodeTable::NodeRelation::Known); + } } } } @@ -786,7 +814,7 @@ void Host::restoreNetwork(bytesConstRef _b) KeyPair Host::networkAlias(bytesConstRef _b) { RLP r(_b); - if (r.itemCount() == 3 && r[0].isInt() && r[0].toInt() == dev::p2p::c_protocolVersion) + if (r.itemCount() == 3 && r[0].isInt() && r[0].toInt() >= 3) return move(KeyPair(move(Secret(r[1].toBytes())))); else return move(KeyPair::create()); diff --git a/libp2p/NodeTable.cpp b/libp2p/NodeTable.cpp index 9d1ba2ba4cc..4be2d9f4d25 100644 --- a/libp2p/NodeTable.cpp +++ b/libp2p/NodeTable.cpp @@ -517,8 +517,19 @@ void NodeTable::onReceived(UDPSocketFace*, bi::udp::endpoint const& _from, bytes case PingNode::type: { PingNode in = PingNode::fromBytesConstRef(_from, rlpBytes); - if (in.version != dev::p2p::c_protocolVersion) - return; + if (in.version < dev::p2p::c_protocolVersion) + { + if (in.version == 3) + { + compat::Pong p(in.source); + p.echo = sha3(rlpBytes); + p.sign(m_secret); + m_socketPointer->send(p); + } + else + return; + } + if (RLPXDatagramFace::secondsSinceEpoch() > in.ts) { clog(NodeTableTriviaSummary) << "Received expired PingNode from " << _from.address().to_string() << ":" << _from.port(); @@ -621,5 +632,27 @@ void PingNode::interpretRLP(bytesConstRef _bytes) ts = r[3].toInt(RLP::Strict); } else - version = 0; + version = r[0].toInt(RLP::Strict); +} + +void Pong::streamRLP(RLPStream& _s) const +{ + _s.appendList(3); + destination.streamRLP(_s); + _s << echo << ts; +} + +void Pong::interpretRLP(bytesConstRef _bytes) +{ + RLP r(_bytes); + destination.interpretRLP(r[0]); + echo = (h256)r[1]; + ts = r[2].toInt(); +} + +void compat::Pong::interpretRLP(bytesConstRef _bytes) +{ + RLP r(_bytes); + echo = (h256)r[0]; + ts = r[1].toInt(); } diff --git a/libp2p/NodeTable.h b/libp2p/NodeTable.h index 864da71349e..be893075ef3 100644 --- a/libp2p/NodeTable.h +++ b/libp2p/NodeTable.h @@ -100,23 +100,15 @@ inline std::ostream& operator<<(std::ostream& _out, NodeTable const& _nodeTable) * NodeTable accepts a port for UDP and will listen to the port on all available * interfaces. * - * - * [Integration] - * @todo TCP endpoints - * @todo GC uniform 1/32 entires at 112500ms interval - * * [Optimization] * @todo serialize evictions per-bucket * @todo store evictions in map, unit-test eviction logic * @todo store root node in table * @todo encapsulate discover into NetworkAlgorithm (task) - * @todo Pong to include ip:port where ping was received * @todo expiration and sha3(id) 'to' for messages which are replies (prevents replay) * @todo cache Ping and FindSelf * * [Networking] - * @todo node-endpoint updates - * @todo TCP endpoints * @todo eth/upnp/natpmp/stun/ice/etc for public-discovery * @todo firewall * @@ -131,7 +123,7 @@ class NodeTable: UDPSocketEvents, public std::enable_shared_from_this using TimePoint = std::chrono::steady_clock::time_point; ///< Steady time point. using NodeIdTimePoint = std::pair; using EvictionTimeout = std::pair; ///< First NodeId (NodeIdTimePoint) may be evicted and replaced with second NodeId. - + public: enum NodeRelation { Unknown = 0, Known }; @@ -149,7 +141,7 @@ class NodeTable: UDPSocketEvents, public std::enable_shared_from_this void processEvents(); /// Add node. Node will be pinged and empty shared_ptr is returned if node has never been seen or NodeId is empty. - std::shared_ptr addNode(Node const& _node, NodeRelation _relation = Unknown); + std::shared_ptr addNode(Node const& _node, NodeRelation _relation = NodeRelation::Unknown); /// To be called when node table is empty. Runs node discovery with m_node.id as the target in order to populate node-table. void discover(); @@ -332,8 +324,8 @@ struct Pong: RLPXDatagram h256 echo; ///< MCD of PingNode uint32_t ts = 0; - void streamRLP(RLPStream& _s) const { _s.appendList(3); destination.streamRLP(_s); _s << echo << ts; } - void interpretRLP(bytesConstRef _bytes) { RLP r(_bytes); destination.interpretRLP(r[0]); echo = (h256)r[1]; ts = r[2].toInt(); } + void streamRLP(RLPStream& _s) const; + void interpretRLP(bytesConstRef _bytes); }; /** @@ -391,6 +383,23 @@ struct Neighbours: RLPXDatagram void streamRLP(RLPStream& _s) const { _s.appendList(2); _s.appendList(neighbours.size()); for (auto& n: neighbours) n.streamRLP(_s); _s << ts; } void interpretRLP(bytesConstRef _bytes) { RLP r(_bytes); for (auto n: r[0]) neighbours.push_back(Neighbour(n)); ts = r[1].toInt(); } }; + +namespace compat +{ + /** + * Pong packet [compatability]: Sent in response to ping + */ + struct Pong: RLPXDatagram + { + Pong(bi::udp::endpoint const& _ep): RLPXDatagram(_ep) {} + Pong(NodeIPEndpoint const& _dest): RLPXDatagram((bi::udp::endpoint)_dest), ts(futureFromEpoch(std::chrono::seconds(60))) {} + static const uint8_t type = 2; + h256 echo; + uint32_t ts = 0; + void streamRLP(RLPStream& _s) const { _s.appendList(2); _s << echo << ts; } + void interpretRLP(bytesConstRef _bytes); + }; +} struct NodeTableWarn: public LogChannel { static const char* name(); static const int verbosity = 0; }; struct NodeTableNote: public LogChannel { static const char* name(); static const int verbosity = 1; }; From 4ac289be47007c6b0a75c8479338ce7c2adb5787 Mon Sep 17 00:00:00 2001 From: subtly Date: Thu, 7 May 2015 00:26:17 +0200 Subject: [PATCH 21/22] fix restore network to support both old and new network file format --- libp2p/Host.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libp2p/Host.cpp b/libp2p/Host.cpp index f781fc1042e..d67dabc4e5d 100644 --- a/libp2p/Host.cpp +++ b/libp2p/Host.cpp @@ -737,6 +737,9 @@ bytes Host::saveNetwork() const void Host::restoreNetwork(bytesConstRef _b) { + if (!_b.size()) + return; + // nodes can only be added if network is added if (!isStarted()) BOOST_THROW_EXCEPTION(NetworkStartRequired()); @@ -756,7 +759,7 @@ void Host::restoreNetwork(bytesConstRef _b) for (auto i: r[2]) { // todo: ipv6 - if (i[0].itemCount() != 4) + if (i[0].itemCount() != 4 && i[0].size() != 4) continue; if (i.itemCount() == 4 || i.itemCount() == 11) From d651501be03f927001ccf0abf6bcc9c75ad20d3c Mon Sep 17 00:00:00 2001 From: subtly Date: Thu, 7 May 2015 15:58:38 +0200 Subject: [PATCH 22/22] Fix network-snapshot method when there's a lot of node activity (dereferencing shared pointer from weak ptr without verifying weak ptr). --- libp2p/NodeTable.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/libp2p/NodeTable.cpp b/libp2p/NodeTable.cpp index 4be2d9f4d25..772ba6147d9 100644 --- a/libp2p/NodeTable.cpp +++ b/libp2p/NodeTable.cpp @@ -138,8 +138,10 @@ list NodeTable::snapshot() const list ret; Guard l(x_state); for (auto s: m_state) - for (auto n: s.nodes) - ret.push_back(*n.lock()); + for (auto np: s.nodes) + if (auto n = np.lock()) + if (!!n) + ret.push_back(*n); return move(ret); }