Dropping ACK packets

[xtexChooser]

strace --trace=%net,%file,%ipc -f

31566 execve("target/debug/peerd", ["target/debug/peerd"], 0x7ffe4fc15ab0 /* 14 vars */) = 0
31566 access("/etc/ld.so.preload", R_OK) = -1 ENOENT (没有那个文件或目录)
31566 openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
31566 newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=137371, ...}, AT_EMPTY_PATH) = 0
31566 openat(AT_FDCWD, "/lib64/libgcc_s.so.1", O_RDONLY|O_CLOEXEC) = 3
31566 newfstatat(3, "", {st_mode=S_IFREG|0755, st_size=145744, ...}, AT_EMPTY_PATH) = 0
31566 openat(AT_FDCWD, "/lib64/libm.so.6", O_RDONLY|O_CLOEXEC) = 3
31566 newfstatat(3, "", {st_mode=S_IFREG|0755, st_size=1013072, ...}, AT_EMPTY_PATH) = 0
31566 openat(AT_FDCWD, "/lib64/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
31566 newfstatat(3, "", {st_mode=S_IFREG|0755, st_size=2339992, ...}, AT_EMPTY_PATH) = 0
31566 openat(AT_FDCWD, "/proc/self/maps", O_RDONLY|O_CLOEXEC) = 3
31566 newfstatat(3, "", {st_mode=S_IFREG|0444, st_size=0, ...}, AT_EMPTY_PATH) = 0
31566 openat(AT_FDCWD, "/proc/self/cgroup", O_RDONLY|O_CLOEXEC) = 3
31566 openat(AT_FDCWD, "/proc/self/mountinfo", O_RDONLY|O_CLOEXEC) = 3
31566 openat(AT_FDCWD, "/sys/fs/cgroup/user.slice/user-1000.slice/user@1000.service/app.slice/app-code\\x2dinsiders-df703ede9d874cdcad076cfd99489807.scope/cpu.max", O_RDONLY|O_CLOEXEC) = -1 ENOENT (没有那个文件或目录)
31566 socketpair(AF_UNIX, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0, [6, 7]) = 0
31566 statx(0, NULL, AT_STATX_SYNC_AS_STAT, STATX_ALL, NULL) = -1 EFAULT (错误的地址)
31566 statx(AT_FDCWD, "peerd.toml", AT_STATX_SYNC_AS_STAT, STATX_ALL, {stx_mask=STATX_ALL|STATX_MNT_ID, stx_attributes=0, stx_mode=S_IFREG|0644, stx_size=352, ...}) = 0
31566 openat(AT_FDCWD, "peerd.toml", O_RDONLY|O_CLOEXEC) = 9
31566 statx(9, "", AT_STATX_SYNC_AS_STAT|AT_EMPTY_PATH, STATX_ALL, {stx_mask=STATX_ALL|STATX_MNT_ID, stx_attributes=0, stx_mode=S_IFREG|0644, stx_size=352, ...}) = 0
31566 socket(AF_NETLINK, SOCK_DGRAM|SOCK_CLOEXEC, NETLINK_ROUTE) = 9
31572 sendto(9, [{nlmsg_len=52, nlmsg_type=0x12 /* NLMSG_??? */, nlmsg_flags=NLM_F_REQUEST, nlmsg_seq=1, nlmsg_pid=0}, "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x13\x00\x03\x00\x70\x65\x65\x72\x34\x32\x64\x38\x37\x66\x37\x65"...], 52, 0, {sa_family=AF_NETLINK, nl_pid=0, nl_groups=00000000}, 12) = 52
31567 recvfrom(9, [{nlmsg_len=1400, nlmsg_type=RTM_NEWLINK, nlmsg_flags=0, nlmsg_seq=1, nlmsg_pid=31566}, {ifi_family=AF_UNSPEC, ifi_type=ARPHRD_NONE, ifi_index=if_nametoindex("peer42d87f7e0c"), ifi_flags=IFF_POINTOPOINT|IFF_NOARP, ifi_change=0}, [[{nla_len=19, nla_type=IFLA_IFNAME}, "peer42d87f7e0c"], [{nla_len=8, nla_type=IFLA_TXQLEN}, 1000], [{nla_len=5, nla_type=IFLA_OPERSTATE}, 2], [{nla_len=5, nla_type=IFLA_LINKMODE}, 0], [{nla_len=8, nla_type=IFLA_MTU}, 1420], [{nla_len=8, nla_type=IFLA_MIN_MTU}, 0], [{nla_len=8, nla_type=IFLA_MAX_MTU}, 2147483552], [{nla_len=8, nla_type=IFLA_GROUP}, 0], [{nla_len=8, nla_type=IFLA_PROMISCUITY}, 0], [{nla_len=8, nla_type=IFLA_NUM_TX_QUEUES}, 1], [{nla_len=8, nla_type=IFLA_GSO_MAX_SEGS}, 65535], [{nla_len=8, nla_type=IFLA_GSO_MAX_SIZE}, 65536], [{nla_len=8, nla_type=IFLA_GRO_MAX_SIZE}, 65536], [{nla_len=8, nla_type=IFLA_TSO_MAX_SIZE}, 65536], [{nla_len=8, nla_type=IFLA_TSO_MAX_SEGS}, 65535], [{nla_len=8, nla_type=IFLA_NUM_RX_QUEUES}, 1], [{nla_len=5, nla_type=IFLA_CARRIER}, 1], [{nla_len=9, nla_type=IFLA_QDISC}, "noop"], [{nla_len=8, nla_type=IFLA_CARRIER_CHANGES}, 0], [{nla_len=8, nla_type=IFLA_CARRIER_UP_COUNT}, 0], [{nla_len=8, nla_type=IFLA_CARRIER_DOWN_COUNT}, 0], [{nla_len=5, nla_type=IFLA_PROTO_DOWN}, 0], [{nla_len=36, nla_type=IFLA_MAP}, {mem_start=0, mem_end=0, base_addr=0, irq=0, dma=0, port=0}], [{nla_len=204, nla_type=IFLA_STATS64}, {rx_packets=0, tx_packets=0, rx_bytes=0, tx_bytes=0, rx_errors=0, tx_errors=0, rx_dropped=0, tx_dropped=0, multicast=0, collisions=0, rx_length_errors=0, rx_over_errors=0, rx_crc_errors=0, rx_frame_errors=0, rx_fifo_errors=0, rx_missed_errors=0, tx_aborted_errors=0, tx_carrier_errors=0, tx_fifo_errors=0, tx_heartbeat_errors=0, tx_window_errors=0, rx_compressed=0, tx_compressed=0, rx_nohandler=0, rx_otherhost_dropped=0}], [{nla_len=100, nla_type=IFLA_STATS}, {rx_packets=0, tx_packets=0, rx_bytes=0, tx_bytes=0, rx_errors=0, tx_errors=0, rx_dropped=0, tx_dropped=0, multicast=0, collisions=0, rx_length_errors=0, rx_over_errors=0, rx_crc_errors=0, rx_frame_errors=0, rx_fifo_errors=0, rx_missed_errors=0, tx_aborted_errors=0, tx_carrier_errors=0, tx_fifo_errors=0, tx_heartbeat_errors=0, tx_window_errors=0, rx_compressed=0, tx_compressed=0, rx_nohandler=0}], [{nla_len=12, nla_type=IFLA_XDP}, [{nla_len=5, nla_type=IFLA_XDP_ATTACHED}, XDP_ATTACHED_NONE]], [{nla_len=20, nla_type=IFLA_LINKINFO}, [{nla_len=14, nla_type=IFLA_INFO_KIND}, "wireguard"]], [{nla_len=804, nla_type=IFLA_AF_SPEC}, [[{nla_len=12, nla_type=AF_MCTP}, [{nla_len=8, nla_type=IFLA_MCTP_NET}, 1]], [{nla_len=140, nla_type=AF_INET}, [{nla_len=136, nla_type=IFLA_INET_CONF}, [[IPV4_DEVCONF_FORWARDING-1]=0, [IPV4_DEVCONF_MC_FORWARDING-1]=0, [IPV4_DEVCONF_PROXY_ARP-1]=0, [IPV4_DEVCONF_ACCEPT_REDIRECTS-1]=0, [IPV4_DEVCONF_SECURE_REDIRECTS-1]=1, [IPV4_DEVCONF_SEND_REDIRECTS-1]=1, [IPV4_DEVCONF_SHARED_MEDIA-1]=1, [IPV4_DEVCONF_RP_FILTER-1]=2, [IPV4_DEVCONF_ACCEPT_SOURCE_ROUTE-1]=0, [IPV4_DEVCONF_BOOTP_RELAY-1]=0, [IPV4_DEVCONF_LOG_MARTIANS-1]=0, [IPV4_DEVCONF_TAG-1]=0, [IPV4_DEVCONF_ARPFILTER-1]=0, [IPV4_DEVCONF_MEDIUM_ID-1]=0, [IPV4_DEVCONF_NOXFRM-1]=0, [IPV4_DEVCONF_NOPOLICY-1]=0, [IPV4_DEVCONF_FORCE_IGMP_VERSION-1]=0, [IPV4_DEVCONF_ARP_ANNOUNCE-1]=0, [IPV4_DEVCONF_ARP_IGNORE-1]=0, [IPV4_DEVCONF_PROMOTE_SECONDARIES-1]=1, [IPV4_DEVCONF_ARP_ACCEPT-1]=0, [IPV4_DEVCONF_ARP_NOTIFY-1]=0, [IPV4_DEVCONF_ACCEPT_LOCAL-1]=0, [IPV4_DEVCONF_SRC_VMARK-1]=0, [IPV4_DEVCONF_PROXY_ARP_PVLAN-1]=0, [IPV4_DEVCONF_ROUTE_LOCALNET-1]=0, [IPV4_DEVCONF_IGMPV2_UNSOLICITED_REPORT_INTERVAL-1]=10000, [IPV4_DEVCONF_IGMPV3_UNSOLICITED_REPORT_INTERVAL-1]=1000, [IPV4_DEVCONF_IGNORE_ROUTES_WITH_LINKDOWN-1]=0, [IPV4_DEVCONF_DROP_UNICAST_IN_L2_MULTICAST-1]=0, [IPV4_DEVCONF_DROP_GRATUITOUS_ARP-1]=0, [IPV4_DEVCONF_BC_FORWARDING-1]=0, ...]]], [{nla_len=648, nla_type=AF_INET6}, [[{nla_len=8, nla_type=IFLA_INET6_FLAGS}, 0], [{nla_len=20, nla_type=IFLA_INET6_CACHEINFO}, {max_reasm_len=65535, tstamp=2174707, reachable_time=16668, retrans_time=1000}], [{nla_len=236, nla_type=IFLA_INET6_CONF}, [[DEVCONF_FORWARDING]=0, [DEVCONF_HOPLIMIT]=64, [DEVCONF_MTU6]=1420, [DEVCONF_ACCEPT_RA]=1, [DEVCONF_ACCEPT_REDIRECTS]=0, [DEVCONF_AUTOCONF]=1, [DEVCONF_DAD_TRANSMITS]=1, [DEVCONF_RTR_SOLICITS]=-1, [DEVCONF_RTR_SOLICIT_INTERVAL]=4000, [DEVCONF_RTR_SOLICIT_DELAY]=1000, [DEVCONF_USE_TEMPADDR]=-1, [DEVCONF_TEMP_VALID_LFT]=604800, [DEVCONF_TEMP_PREFERED_LFT]=86400, [DEVCONF_REGEN_MAX_RETRY]=3, [DEVCONF_MAX_DESYNC_FACTOR]=600, [DEVCONF_MAX_ADDRESSES]=16, [DEVCONF_FORCE_MLD_VERSION]=0, [DEVCONF_ACCEPT_RA_DEFRTR]=1, [DEVCONF_ACCEPT_RA_PINFO]=1, [DEVCONF_ACCEPT_RA_RTR_PREF]=1, [DEVCONF_RTR_PROBE_INTERVAL]=60000, [DEVCONF_ACCEPT_RA_RT_INFO_MAX_PLEN]=0, [DEVCONF_PROXY_NDP]=0, [DEVCONF_OPTIMISTIC_DAD]=0, [DEVCONF_ACCEPT_SOURCE_ROUTE]=0, [DEVCONF_MC_FORWARDING]=0, [DEVCONF_DISABLE_IPV6]=0, [DEVCONF_ACCEPT_DAD]=-1, [DEVCONF_FORCE_TLLAO]=0, [DEVCONF_NDISC_NOTIFY]=0, [DEVCONF_MLDV1_UNSOLICITED_REPORT_INTERVAL]=10000, [DEVCONF_MLDV2_UNSOLICITED_REPORT_INTERVAL]=1000, ...]], [{nla_len=300, nla_type=IFLA_INET6_STATS}, [[IPSTATS_MIB_NUM]=37, [IPSTATS_MIB_INPKTS]=0, [IPSTATS_MIB_INOCTETS]=0, [IPSTATS_MIB_INDELIVERS]=0, [IPSTATS_MIB_OUTFORWDATAGRAMS]=0, [IPSTATS_MIB_OUTPKTS]=0, [IPSTATS_MIB_OUTOCTETS]=0, [IPSTATS_MIB_INHDRERRORS]=0, [IPSTATS_MIB_INTOOBIGERRORS]=0, [IPSTATS_MIB_INNOROUTES]=0, [IPSTATS_MIB_INADDRERRORS]=0, [IPSTATS_MIB_INUNKNOWNPROTOS]=0, [IPSTATS_MIB_INTRUNCATEDPKTS]=0, [IPSTATS_MIB_INDISCARDS]=0, [IPSTATS_MIB_OUTDISCARDS]=0, [IPSTATS_MIB_OUTNOROUTES]=0, [IPSTATS_MIB_REASMTIMEOUT]=0, [IPSTATS_MIB_REASMREQDS]=0, [IPSTATS_MIB_REASMOKS]=0, [IPSTATS_MIB_REASMFAILS]=0, [IPSTATS_MIB_FRAGOKS]=0, [IPSTATS_MIB_FRAGFAILS]=0, [IPSTATS_MIB_FRAGCREATES]=0, [IPSTATS_MIB_INMCASTPKTS]=0, [IPSTATS_MIB_OUTMCASTPKTS]=0, [IPSTATS_MIB_INBCASTPKTS]=0, [IPSTATS_MIB_OUTBCASTPKTS]=0, [IPSTATS_MIB_INMCASTOCTETS]=0, [IPSTATS_MIB_OUTMCASTOCTETS]=0, [IPSTATS_MIB_INBCASTOCTETS]=0, [IPSTATS_MIB_OUTBCASTOCTETS]=0, [IPSTATS_MIB_CSUMERRORS]=0, ...]], [{nla_len=52, nla_type=IFLA_INET6_ICMP6STATS}, [[ICMP6_MIB_NUM]=6, [ICMP6_MIB_INMSGS]=0, [ICMP6_MIB_INERRORS]=0, [ICMP6_MIB_OUTMSGS]=0, [ICMP6_MIB_OUTERRORS]=0, [ICMP6_MIB_CSUMERRORS]=0]], [{nla_len=20, nla_type=IFLA_INET6_TOKEN}, inet_pton(AF_INET6, "::")], [{nla_len=5, nla_type=IFLA_INET6_ADDR_GEN_MODE}, IN6_ADDR_GEN_MODE_EUI64]]]]]]], 65536, 0, {sa_family=AF_NETLINK, nl_pid=0, nl_groups=00000000}, [12]) = 1400
31567 recvfrom(9, 0x5625a56b11d0, 65536, 0, 0x7f6d82e464f8, [12]) = -1 EAGAIN (资源暂时不可用)
31566 socket(AF_NETLINK, SOCK_DGRAM|SOCK_CLOEXEC, NETLINK_GENERIC) = 10
31571 sendto(10, [{nlmsg_len=36, nlmsg_type=0x10 /* NLMSG_??? */, nlmsg_flags=NLM_F_REQUEST, nlmsg_seq=1, nlmsg_pid=0}, "\x03\x02\x00\x00\x0e\x00\x02\x00\x77\x69\x72\x65\x67\x75\x61\x72\x64\x00\x00\x00"], 36, 0, {sa_family=AF_NETLINK, nl_pid=0, nl_groups=00000000}, 12) = 36
31567 recvfrom(10, [{nlmsg_len=112, nlmsg_type=nlctrl, nlmsg_flags=0, nlmsg_seq=1, nlmsg_pid=31566}, "\x01\x02\x00\x00\x0e\x00\x02\x00\x77\x69\x72\x65\x67\x75\x61\x72\x64\x00\x00\x00\x06\x00\x01\x00\x23\x00\x00\x00\x08\x00\x03\x00"...], 65536, 0, {sa_family=AF_NETLINK, nl_pid=0, nl_groups=00000000}, [12]) = 112
31567 recvfrom(10, 0x5625a56c3340, 65536, 0, 0x7f6d82e46598, [12]) = -1 EAGAIN (资源暂时不可用)
31567 sendto(10, [{nlmsg_len=224, nlmsg_type=wireguard, nlmsg_flags=NLM_F_REQUEST|NLM_F_ACK, nlmsg_seq=2, nlmsg_pid=0}, "\x01\x01\x00\x00\x13\x00\x02\x00\x70\x65\x65\x72\x34\x32\x64\x38\x37\x66\x37\x65\x30\x63\x00\x00\x24\x00\x03\x00\x40\xc4\xec\x33"...], 224, 0, {sa_family=AF_NETLINK, nl_pid=0, nl_groups=00000000}, 12) = 224
31571 recvfrom(10, [{nlmsg_len=36, nlmsg_type=NLMSG_ERROR, nlmsg_flags=NLM_F_CAPPED, nlmsg_seq=2, nlmsg_pid=31566}, {error=0, msg={nlmsg_len=224, nlmsg_type=wireguard, nlmsg_flags=NLM_F_REQUEST|NLM_F_ACK, nlmsg_seq=2, nlmsg_pid=0}}], 65536, 0, {sa_family=AF_NETLINK, nl_pid=0, nl_groups=00000000}, [12]) = 36
31571 recvfrom(10, 0x5625a56c3340, 65536, 0, 0x7f6d82843598, [12]) = -1 EAGAIN (资源暂时不可用)
31572 +++ exited with 0 +++
31570 +++ exited with 0 +++
31567 +++ exited with 0 +++
31571 +++ exited with 0 +++
31568 +++ exited with 0 +++
31569 +++ exited with 0 +++
31566 +++ exited with 101 +++

At:

31567 sendto(10, [{nlmsg_len=224, nlmsg_type=wireguard, nlmsg_flags=NLM_F_REQUEST|NLM_F_ACK, nlmsg_seq=2, nlmsg_pid=0}, "\x01\x01\x00\x00\x13\x00\x02\x00\x70\x65\x65\x72\x34\x32\x64\x38\x37\x66\x37\x65\x30\x63\x00\x00\x24\x00\x03\x00\x40\xc4\xec\x33"...], 224, 0, {sa_family=AF_NETLINK, nl_pid=0, nl_groups=00000000}, 12) = 224
31571 recvfrom(10, [{nlmsg_len=36, nlmsg_type=NLMSG_ERROR, nlmsg_flags=NLM_F_CAPPED, nlmsg_seq=2, nlmsg_pid=31566}, {error=0, msg={nlmsg_len=224, nlmsg_type=wireguard, nlmsg_flags=NLM_F_REQUEST|NLM_F_ACK, nlmsg_seq=2, nlmsg_pid=0}}], 65536, 0, {sa_family=AF_NETLINK, nl_pid=0, nl_groups=00000000}, [12]) = 36

The code is here. After I sent a Wireguard SET_DEVICE, I called await? to get response but got None. However, the strace shows the call finished. I tryed some other parameters even got a EINVAL but still None.

I tested the example in netlink-packet-wireguard, it works.

But when I apply this diff, the example failed:

diff --git a/examples/get_wireguard_info.rs b/examples/get_wireguard_info.rs
index c410997..9b4d692 100644
--- a/examples/get_wireguard_info.rs
+++ b/examples/get_wireguard_info.rs
@@ -3,7 +3,7 @@
 use futures::StreamExt;
 use genetlink::new_connection;
 use netlink_packet_core::{
-    NetlinkMessage, NetlinkPayload, NLM_F_DUMP, NLM_F_REQUEST,
+    NetlinkMessage, NetlinkPayload, NLM_F_ACK, NLM_F_REQUEST,
 };
 use netlink_packet_generic::GenlMessage;
 use netlink_packet_wireguard::{
@@ -27,11 +27,11 @@ async fn main() {
 
     let genlmsg: GenlMessage<Wireguard> =
         GenlMessage::from_payload(Wireguard {
-            cmd: WireguardCmd::GetDevice,
+            cmd: WireguardCmd::SetDevice,
             nlas: vec![WgDeviceAttrs::IfName(argv[1].clone())],
         });
     let mut nlmsg = NetlinkMessage::from(genlmsg);
-    nlmsg.header.flags = NLM_F_REQUEST | NLM_F_DUMP;
+    nlmsg.header.flags = NLM_F_REQUEST | NLM_F_ACK;
 
     let mut res = handle.request(nlmsg).await.unwrap();
 
@@ -44,7 +44,7 @@ async fn main() {
             NetlinkPayload::Error(e) => {
                 eprintln!("Error: {:?}", e.to_io());
             }
-            _ => (),
+            _ => println!("ukn rx"),
         };
     }
 }

[pid  5543] sendto(6, [{nlmsg_len=32, nlmsg_type=wireguard, nlmsg_flags=NLM_F_REQUEST|NLM_F_ACK, nlmsg_seq=2, nlmsg_pid=0}, "\x01\x01\x00\x00\x0b\x00\x02\x00\x53\x65\x72\x76\x65\x72\x00\x00"], 32, 0, {sa_family=AF_NETLINK, nl_pid=0, nl_groups=00000000}, 12) = 32
[pid  5547] recvfrom(6, [{nlmsg_len=36, nlmsg_type=NLMSG_ERROR, nlmsg_flags=NLM_F_CAPPED, nlmsg_seq=2, nlmsg_pid=5541}, {error=0, msg={nlmsg_len=32, nlmsg_type=wireguard, nlmsg_flags=NLM_F_REQUEST|NLM_F_ACK, nlmsg_seq=2, nlmsg_pid=0}}], 65536, 0, {sa_family=AF_NETLINK, nl_pid=0, nl_groups=00000000}, [12]) = 36
[pid  5547] recvfrom(6, 0x55cc5a7f1270, 65536, 0, 0x7f3ed5b6f5c8, [12]) = -1 EAGAIN (资源暂时不可用)

With this patch, if a ACK message is received, ukn rx should be printed, but it does not.

[xtexChooser]

It seems that netlink-packet-core/src/message.rs got the ACK correctly but it got dropped later.

[xtexChooser]

I replaced that line in message.rs to let the ACK be returned as an error and the error get returned. When I change it to Done or Noop, the message seems to be dropped in some place.

[xtexChooser]

Ops, the ACK is dropped in netlink-proto: src/connection.rs#L228.

[xtexChooser]

The ACK message is very important for WG operations. I think there should be at least a option to make the ACK forwarded.

[cathay4t]

rust-netlink/netlink-proto#22 is created for this request. Please check there. Thank you!