Merge branch 'bitcoin' into auxpow

.cirrus.yml

@@ -182,6 +182,7 @@ task:
   persistent_worker:
     labels:
       type: medium
+  timeout_in: 240m  # larger timeout, due to the high CPU demand
   env:
     FILE_ENV: "./ci/test/00_setup_env_native_fuzz.sh"
 

.tx/config

@@ -1,7 +1,7 @@
 [main]
 host = https://www.transifex.com
 
-[o:bitcoin:p:bitcoin:r:qt-translation-028x]
+[o:bitcoin:p:bitcoin:r:qt-translation-029x]
 file_filter = src/qt/locale/bitcoin_<lang>.xlf
 source_file = src/qt/locale/bitcoin_en.xlf
 source_lang = en

ci/lint/06_script.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2018-2022 The Bitcoin Core developers
+# Copyright (c) 2018-present The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
@@ -9,23 +9,13 @@ export LC_ALL=C
 set -ex
 
 if [ -n "$CIRRUS_PR" ]; then
-  COMMIT_RANGE="HEAD~..HEAD"
+  export COMMIT_RANGE="HEAD~..HEAD"
   if [ "$(git rev-list -1 HEAD)" != "$(git rev-list -1 --merges HEAD)" ]; then
     echo "Error: The top commit must be a merge commit, usually the remote 'pull/${PR_NUMBER}/merge' branch."
     false
   fi
-else
-  # Otherwise, assume that a merge commit exists. This merge commit is assumed
-  # to be the base, after which linting will be done. If the merge commit is
-  # HEAD, the range will be empty.
-  COMMIT_RANGE="$( git rev-list --max-count=1 --merges HEAD )..HEAD"
 fi
-export COMMIT_RANGE
 
-echo
-git log --no-merges --oneline "$COMMIT_RANGE"
-echo
-test/lint/commit-script-check.sh "$COMMIT_RANGE"
 RUST_BACKTRACE=1 "${LINT_RUNNER_PATH}/test_runner"
 
 if [ "$CIRRUS_REPO_FULL_NAME" = "bitcoin/bitcoin" ] && [ "$CIRRUS_PR" = "" ] ; then

ci/test/00_setup_env_i686_multiprocess.sh

@@ -18,7 +18,6 @@ export BITCOIN_CONFIG="\
  -DCMAKE_BUILD_TYPE=Debug \
  -DCMAKE_C_COMPILER='clang;-m32' \
  -DCMAKE_CXX_COMPILER='clang++;-m32' \
- -DCMAKE_CXX_FLAGS='-Wno-error=documentation' \
  -DAPPEND_CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE' \
 "
 export BITCOIND=bitcoin-node  # Used in functional tests

contrib/tracing/README.md

@@ -335,4 +335,25 @@ $ python3 contrib/tracing/mempool_monitor.py $(pidof bitcoind)
  │ 13:10:32Z added c78e87be86c828137a6e7e00a177c03b52202ce4c39029b99904c2a094b9da87 with feerate 11.00 sat/vB (1562 sat, 142 vbytes)                                            │
  │                                                                                                                                                                              │
  └──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
+
+### log_p2p_connections.bt
+
+A `bpftrace` script to log information about opened, closed, misbehaving, and
+evicted P2P connections. Uses the `net:*_connection` tracepoints.
+
+```bash
+$ bpftrace contrib/tracing/log_p2p_connections.bt
+```
+
+This should produce an output similar to the following.
+
+```bash
+Attaching 6 probes...
+Logging opened, closed, misbehaving, and evicted P2P connections
+OUTBOUND conn to 127.0.0.1:15287: id=0, type=block-relay-only, network=0, total_out=1
+INBOUND conn from 127.0.0.1:45324: id=1, type=inbound, network=0, total_in=1
+MISBEHAVING conn id=1, score_before=0, score_increase=20, message='getdata message size = 50001', threshold_exceeded=false
+CLOSED conn to 127.0.0.1:15287: id=0, type=block-relay-only, network=0, established=1231006505
+EVICTED conn to 127.0.0.1:45324: id=1, type=inbound, network=0, established=1612312312
+...
 ```

contrib/tracing/log_p2p_connections.bt

@@ -0,0 +1,51 @@
+#!/usr/bin/env bpftrace
+
+BEGIN
+{
+  printf("Logging opened, closed, misbehaving, and evicted P2P connections\n")
+}
+
+usdt:./build/src/bitcoind:net:inbound_connection
+{
+  $id = (int64) arg0;
+  $addr = str(arg1);
+  $conn_type = str(arg2);
+  $network = (int32) arg3;
+  $existing = (uint64) arg4;
+  printf("INBOUND conn from %s: id=%ld, type=%s, network=%d, total=%d\n", $addr, $id, $conn_type, $network, $existing);
+}
+
+usdt:./build/src/bitcoind:net:outbound_connection
+{
+  $id = (int64) arg0;
+  $addr = str(arg1);
+  $conn_type = str(arg2);
+  $network = (int32) arg3;
+  $existing = (uint64) arg4;
+  printf("OUTBOUND conn to %s: id=%ld, type=%s, network=%d, total=%d\n", $addr, $id, $conn_type, $network, $existing);
+}
+
+usdt:./build/src/bitcoind:net:closed_connection
+{
+  $id = (int64) arg0;
+  $addr = str(arg1);
+  $conn_type = str(arg2);
+  $network = (int32) arg3;
+  printf("CLOSED conn to %s: id=%ld, type=%s, network=%d, established=%ld\n", $addr, $id, $conn_type, $network, arg4);
+}
+
+usdt:./build/src/bitcoind:net:evicted_inbound_connection
+{
+  $id = (int64) arg0;
+  $addr = str(arg1);
+  $conn_type = str(arg2);
+  $network = (int32) arg3;
+  printf("EVICTED conn to %s: id=%ld, type=%s, network=%d, established=%ld\n", $addr, $id, $conn_type, $network, arg4);
+}
+
+usdt:./build/src/bitcoind:net:misbehaving_connection
+{
+  $id = (int64) arg0;
+  $message = str(arg1);
+  printf("MISBEHAVING conn id=%ld, message='%s'\n", $id, $message);
+}

depends/Makefile

@@ -217,7 +217,6 @@ $(host_prefix)/toolchain.cmake : toolchain.cmake.in $(host_prefix)/.stamp_$(fina
             -e 's|@STRIP@|$(host_STRIP)|' \
             -e 's|@OBJCOPY@|$(host_OBJCOPY)|' \
             -e 's|@OBJDUMP@|$(host_OBJDUMP)|' \
-            -e 's|@depends_prefix@|$(host_prefix)|' \
             -e 's|@CFLAGS@|$(strip $(host_CFLAGS))|' \
             -e 's|@CFLAGS_RELEASE@|$(strip $(host_release_CFLAGS))|' \
             -e 's|@CFLAGS_DEBUG@|$(strip $(host_debug_CFLAGS))|' \

depends/packages/capnp.mk

@@ -9,7 +9,7 @@ define $(package)_set_vars :=
   $(package)_config_opts := -DBUILD_TESTING=OFF
   $(package)_config_opts += -DWITH_OPENSSL=OFF
   $(package)_config_opts += -DWITH_ZLIB=OFF
-  $(package)_cxxflags += -ffile-prefix-map=$$($(package)_extract_dir)=/usr
+  $(package)_cxxflags += -fdebug-prefix-map=$($(package)_extract_dir)=/usr -fmacro-prefix-map=$($(package)_extract_dir)=/usr
 endef
 
 define $(package)_config_cmds

depends/packages/libevent.mk

@@ -13,7 +13,7 @@ define $(package)_set_vars
   $(package)_config_opts=-DCMAKE_BUILD_TYPE=None -DEVENT__DISABLE_BENCHMARK=ON -DEVENT__DISABLE_OPENSSL=ON
   $(package)_config_opts+=-DEVENT__DISABLE_SAMPLES=ON -DEVENT__DISABLE_REGRESS=ON
   $(package)_config_opts+=-DEVENT__DISABLE_TESTS=ON -DEVENT__LIBRARY_TYPE=STATIC
-  $(package)_cflags += -ffile-prefix-map=$($(package)_extract_dir)=/usr
+  $(package)_cflags += -fdebug-prefix-map=$($(package)_extract_dir)=/usr -fmacro-prefix-map=$($(package)_extract_dir)=/usr
   $(package)_cppflags += -D_GNU_SOURCE
   $(package)_cppflags_mingw32=-D_WIN32_WINNT=0x0A00
 

depends/packages/libmultiprocess.mk

@@ -13,7 +13,7 @@ ifneq ($(host),$(build))
 $(package)_config_opts := -DCAPNP_EXECUTABLE="$$(native_capnp_prefixbin)/capnp"
 $(package)_config_opts += -DCAPNPC_CXX_EXECUTABLE="$$(native_capnp_prefixbin)/capnpc-c++"
 endif
-$(package)_cxxflags += -ffile-prefix-map=$$($(package)_extract_dir)=/usr
+$(package)_cxxflags += -fdebug-prefix-map=$($(package)_extract_dir)=/usr -fmacro-prefix-map=$($(package)_extract_dir)=/usr
 endef
 
 define $(package)_config_cmds

depends/packages/zeromq.mk

@@ -17,7 +17,7 @@ define $(package)_set_vars
   $(package)_config_opts += -DWITH_LIBBSD=OFF -DENABLE_CURVE=OFF -DENABLE_CPACK=OFF
   $(package)_config_opts += -DBUILD_SHARED=OFF -DBUILD_TESTS=OFF -DZMQ_BUILD_TESTS=OFF
   $(package)_config_opts += -DENABLE_DRAFTS=OFF -DZMQ_BUILD_TESTS=OFF
-  $(package)_cxxflags += -ffile-prefix-map=$($(package)_extract_dir)=/usr
+  $(package)_cxxflags += -fdebug-prefix-map=$($(package)_extract_dir)=/usr -fmacro-prefix-map=$($(package)_extract_dir)=/usr
   $(package)_config_opts_mingw32 += -DZMQ_WIN32_WINNT=0x0A00 -DZMQ_HAVE_IPC=OFF
 endef
 

depends/toolchain.cmake.in

@@ -81,12 +81,12 @@ set(CMAKE_OBJDUMP "@OBJDUMP@")
 # affected by a potentially random environment.
 set(CMAKE_FIND_USE_CMAKE_ENVIRONMENT_PATH OFF)
 
-set(CMAKE_FIND_ROOT_PATH "@depends_prefix@")
+set(CMAKE_FIND_ROOT_PATH "${CMAKE_CURRENT_LIST_DIR}")
 set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
 set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
 set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
 set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)
-set(QT_TRANSLATIONS_DIR "@depends_prefix@/translations")
+set(QT_TRANSLATIONS_DIR "${CMAKE_CURRENT_LIST_DIR}/translations")
 
 if(CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND NOT CMAKE_HOST_APPLE)
   # The find_package(Qt ...) function internally uses find_library()

doc/tracing.md

@@ -55,14 +55,17 @@ The currently available tracepoints are listed here.
 
 ### Context `net`
 
+[^address-length]: An Onion v3 address with a `:` and a five digit port has 68
+  chars. However, addresses of peers added with host names might be longer.
+
 #### Tracepoint `net:inbound_message`
 
 Is called when a message is received from a peer over the P2P network. Passes
 information about our peer, the connection and the message as arguments.
 
 Arguments passed:
 1. Peer ID as `int64`
-2. Peer Address and Port (IPv4, IPv6, Tor v3, I2P, ...) as `pointer to C-style String` (max. length 68 characters)
+2. Peer Address and Port (IPv4, IPv6, Tor v3, I2P, ...) as `pointer to C-style String` (normally up to 68 characters[^address-length])
 3. Connection Type (inbound, feeler, outbound-full-relay, ...) as `pointer to C-style String` (max. length 20 characters)
 4. Message Type (inv, ping, getdata, addrv2, ...) as `pointer to C-style String` (max. length 20 characters)
 5. Message Size in bytes as `uint64`
@@ -81,7 +84,7 @@ information about our peer, the connection and the message as arguments.
 
 Arguments passed:
 1. Peer ID as `int64`
-2. Peer Address and Port (IPv4, IPv6, Tor v3, I2P, ...) as `pointer to C-style String` (max. length 68 characters)
+2. Peer Address and Port (IPv4, IPv6, Tor v3, I2P, ...) as `pointer to C-style String` (normally up to 68 characters[^address-length])
 3. Connection Type (inbound, feeler, outbound-full-relay, ...) as `pointer to C-style String` (max. length 20 characters)
 4. Message Type (inv, ping, getdata, addrv2, ...) as `pointer to C-style String` (max. length 20 characters)
 5. Message Size in bytes as `uint64`
@@ -93,6 +96,62 @@ to user-space in full. Messages longer than a 32kb might be cut off. This can
 be detected in tracing scripts by comparing the message size to the length of
 the passed message.
 
+#### Tracepoint `net:inbound_connection`
+
+Is called when a new inbound connection is opened to us. Passes information about
+the peer and the number of inbound connections including the newly opened connection.
+
+Arguments passed:
+1. Peer ID as `int64`
+2. Peer address and port (IPv4, IPv6, Tor v3, I2P, ...) as `pointer to C-style String` (normally up to 68 characters[^address-length])
+3. Connection Type (inbound, feeler, outbound-full-relay, ...) as `pointer to C-style String` (max. length 20 characters)
+4. Network the peer connects from as `uint32` (1 = IPv4, 2 = IPv6, 3 = Onion, 4 = I2P, 5 = CJDNS). See `Network` enum in `netaddress.h`.
+5. Number of existing inbound connections as `uint64` including the newly opened inbound connection.
+
+#### Tracepoint `net:outbound_connection`
+
+Is called when a new outbound connection is opened by us. Passes information about
+the peer and the number of outbound connections including the newly opened connection.
+
+Arguments passed:
+1. Peer ID as `int64`
+2. Peer address and port (IPv4, IPv6, Tor v3, I2P, ...) as `pointer to C-style String` (normally up to 68 characters[^address-length])
+3. Connection Type (inbound, feeler, outbound-full-relay, ...) as `pointer to C-style String` (max. length 20 characters)
+4. Network of the peer as `uint32` (1 = IPv4, 2 = IPv6, 3 = Onion, 4 = I2P, 5 = CJDNS). See `Network` enum in `netaddress.h`.
+5. Number of existing outbound connections as `uint64` including the newly opened outbound connection.
+
+#### Tracepoint `net:evicted_inbound_connection`
+
+Is called when a inbound connection is evicted by us. Passes information about the evicted peer and the time at connection establishment.
+
+Arguments passed:
+1. Peer ID as `int64`
+2. Peer address and port (IPv4, IPv6, Tor v3, I2P, ...) as `pointer to C-style String` (normally up to 68 characters[^address-length])
+3. Connection Type (inbound, feeler, outbound-full-relay, ...) as `pointer to C-style String` (max. length 20 characters)
+4. Network the peer connects from as `uint32` (1 = IPv4, 2 = IPv6, 3 = Onion, 4 = I2P, 5 = CJDNS). See `Network` enum in `netaddress.h`.
+5. Connection established UNIX epoch timestamp in seconds as `uint64`.
+
+#### Tracepoint `net:misbehaving_connection`
+
+Is called when a connection is misbehaving. Passes the peer id and a
+reason for the peers misbehavior.
+
+Arguments passed:
+1. Peer ID as `int64`.
+2. Reason why the peer is misbehaving as `pointer to C-style String` (max. length 128 characters).
+
+#### Tracepoint `net:closed_connection`
+
+Is called when a connection is closed. Passes information about the closed peer
+and the time at connection establishment.
+
+Arguments passed:
+1. Peer ID as `int64`
+2. Peer address and port (IPv4, IPv6, Tor v3, I2P, ...) as `pointer to C-style String` (normally up to 68 characters[^address-length])
+3. Connection Type (inbound, feeler, outbound-full-relay, ...) as `pointer to C-style String` (max. length 20 characters)
+4. Network the peer connects from as `uint32` (1 = IPv4, 2 = IPv6, 3 = Onion, 4 = I2P, 5 = CJDNS). See `Network` enum in `netaddress.h`.
+5. Connection established UNIX epoch timestamp in seconds as `uint64`.
+
 ### Context `validation`
 
 #### Tracepoint `validation:block_connected`

src/net.cpp

@@ -53,6 +53,10 @@
 #include <optional>
 #include <unordered_map>
 
+TRACEPOINT_SEMAPHORE(net, closed_connection);
+TRACEPOINT_SEMAPHORE(net, evicted_inbound_connection);
+TRACEPOINT_SEMAPHORE(net, inbound_connection);
+TRACEPOINT_SEMAPHORE(net, outbound_connection);
 TRACEPOINT_SEMAPHORE(net, outbound_message);
 
 /** Maximum number of block-relay-only anchor connections */
@@ -560,6 +564,13 @@ void CNode::CloseSocketDisconnect()
     if (m_sock) {
         LogDebug(BCLog::NET, "Resetting socket for peer=%d%s", GetId(), LogIP(fLogIPs));
         m_sock.reset();
+
+        TRACEPOINT(net, closed_connection,
+            GetId(),
+            m_addr_name.c_str(),
+            ConnectionTypeAsString().c_str(),
+            ConnectedThroughNetwork(),
+            Ticks<std::chrono::seconds>(m_connected));
     }
     m_i2p_sam_session.reset();
 }
@@ -1708,6 +1719,12 @@ bool CConnman::AttemptToEvictConnection()
     for (CNode* pnode : m_nodes) {
         if (pnode->GetId() == *node_id_to_evict) {
             LogDebug(BCLog::NET, "selected %s connection for eviction, %s", pnode->ConnectionTypeAsString(), pnode->DisconnectMsg(fLogIPs));
+            TRACEPOINT(net, evicted_inbound_connection,
+                pnode->GetId(),
+                pnode->m_addr_name.c_str(),
+                pnode->ConnectionTypeAsString().c_str(),
+                pnode->ConnectedThroughNetwork(),
+                Ticks<std::chrono::seconds>(pnode->m_connected));
             pnode->fDisconnect = true;
             return true;
         }
@@ -1833,6 +1850,12 @@ void CConnman::CreateNodeFromAcceptedSocket(std::unique_ptr<Sock>&& sock,
         m_nodes.push_back(pnode);
     }
     LogDebug(BCLog::NET, "connection from %s accepted\n", addr.ToStringAddrPort());
+    TRACEPOINT(net, inbound_connection,
+        pnode->GetId(),
+        pnode->m_addr_name.c_str(),
+        pnode->ConnectionTypeAsString().c_str(),
+        pnode->ConnectedThroughNetwork(),
+        GetNodeCount(ConnectionDirection::In));
 
     // We received a new connection, harvest entropy from the time (and our peer count)
     RandAddEvent((uint32_t)id);
@@ -2995,6 +3018,13 @@ void CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFai
         // update connection count by network
         if (pnode->IsManualOrFullOutboundConn()) ++m_network_conn_counts[pnode->addr.GetNetwork()];
     }
+
+    TRACEPOINT(net, outbound_connection,
+        pnode->GetId(),
+        pnode->m_addr_name.c_str(),
+        pnode->ConnectionTypeAsString().c_str(),
+        pnode->ConnectedThroughNetwork(),
+        GetNodeCount(ConnectionDirection::Out));
 }
 
 Mutex NetEventsInterface::g_msgproc_mutex;

src/net_processing.cpp

@@ -58,6 +58,7 @@
 using namespace util::hex_literals;
 
 TRACEPOINT_SEMAPHORE(net, inbound_message);
+TRACEPOINT_SEMAPHORE(net, misbehaving_connection);
 
 /** Headers download timeout.
  *  Timeout = base + per_header * (expected number of headers) */
@@ -1756,6 +1757,10 @@ void PeerManagerImpl::Misbehaving(Peer& peer, const std::string& message)
     const std::string message_prefixed = message.empty() ? "" : (": " + message);
     peer.m_should_discourage = true;
     LogDebug(BCLog::NET, "Misbehaving: peer=%d%s\n", peer.m_id, message_prefixed);
+    TRACEPOINT(net, misbehaving_connection,
+        peer.m_id,
+        message.c_str()
+    );
 }
 
 void PeerManagerImpl::MaybePunishNodeForBlock(NodeId nodeid, const BlockValidationState& state,