From 299862ed9870fc1cf426efc33a6b1561058cdfaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jarom=C3=ADr=20Smr=C4=8Dek?= <4plague@gmail.com> Date: Tue, 30 Jul 2024 15:59:29 +0200 Subject: [PATCH 1/3] Disable hardware-tests support for pytest --- src/dp_port.c | 5 ----- src/dp_service.c | 9 --------- test/local/dp_service.py | 1 + 3 files changed, 1 insertion(+), 14 deletions(-) diff --git a/src/dp_port.c b/src/dp_port.c index 04414188..3c4327c3 100644 --- a/src/dp_port.c +++ b/src/dp_port.c @@ -613,13 +613,8 @@ static int dp_init_port(struct dp_port *port) } if (dp_conf_is_offload_enabled()) { -#ifdef ENABLE_PYTEST - if (port->peer_pf_port_id != dp_get_pf1()->port_id) -#endif - { if (DP_FAILED(dp_port_bind_port_hairpins(port))) return DP_ERROR; - } if (!port->is_pf) if (DP_FAILED(dp_install_vf_init_rte_rules(port))) diff --git a/src/dp_service.c b/src/dp_service.c index f801dcd3..05caf282 100644 --- a/src/dp_service.c +++ b/src/dp_service.c @@ -173,17 +173,8 @@ static int init_interfaces(void) if (DP_FAILED(dp_start_port(dp_get_port_by_pf_index(0)))) return DP_ERROR; - // PF1 is always started (can receive from outside) even when not used for Tx - // but test suite can use it for monitoring in some setups, so do not bind it then -#ifdef ENABLE_PYTEST -#ifndef ENABLE_PF1_PROXY - if (dp_conf_is_wcmp_enabled()) -#endif -#endif - { if (DP_FAILED(dp_start_port(dp_get_port_by_pf_index(1)))) return DP_ERROR; - } #ifdef ENABLE_PF1_PROXY if (DP_FAILED(dp_start_pf_proxy_tap_port())) diff --git a/test/local/dp_service.py b/test/local/dp_service.py index 0749c241..d054b167 100755 --- a/test/local/dp_service.py +++ b/test/local/dp_service.py @@ -24,6 +24,7 @@ def __init__(self, build_path, port_redundancy, fast_flow_timeout, self.hardware = hardware if self.hardware: + raise ValueError("Hardware tests are currently not supported") if self.port_redundancy: raise ValueError("Port redundancy is not supported when testing on actual hardware") self.reconfigure_tests(DpService.DP_SERVICE_CONF) From b8ce1ca3c988655544558786bf83bd5b0c8ee595 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jarom=C3=ADr=20Smr=C4=8Dek?= <4plague@gmail.com> Date: Thu, 1 Aug 2024 16:18:15 +0200 Subject: [PATCH 2/3] Add Mellanox-enabled testing to pytest suite using packet reflection --- docs/testing/README.md | 3 ++ docs/testing/mellanox.md | 8 ++-- test/local/dp_service.py | 26 ++++++++----- test/local/helpers.py | 39 ++++++++++++++++++- test/local/reflector.py | 75 ++++++++++++++++++++++++++++++++++++ test/local/test_flows.py | 10 ++--- test/local/test_lb.py | 51 +++++++----------------- test/local/test_nat.py | 10 +---- test/local/test_telemetry.py | 47 +++++++++++++++++++--- test/local/test_virtsvc.py | 14 +++++-- 10 files changed, 210 insertions(+), 73 deletions(-) create mode 100755 test/local/reflector.py diff --git a/docs/testing/README.md b/docs/testing/README.md index f3a042fd..2bbf3129 100644 --- a/docs/testing/README.md +++ b/docs/testing/README.md @@ -31,6 +31,9 @@ When running `pytest` directly in the `test/` directory, only a specific set of If one should want to instead run your own `dpservice-bin` instance (e.g. for running under a debugger), the `--attach` argument connects to an already running service instead of starting its own (which in turn can be started via a helper `dp_service.py` script). This comes with the caveat of ensuring the right arguments are passed to the service at startup. +#### Pytest on Mellanox +By default, `pytest` runs using virtual intefaces (TAPs). By providing `--hw` command-line option, it can instead use real NIC based on the contents of `/tmp/dp_service.conf`. For more information [see Mellanox testing guide](mellanox.md#two-machine-setup). + ## Docker There is a tester image provided by this repo, simply build it using this repo's `Dockerfile` and use `--target tester`. For fully working images a GitHub PAT is needed: `docker build --secret=id=github_token,src= --target tester .` diff --git a/docs/testing/mellanox.md b/docs/testing/mellanox.md index bec7510b..ec9d97a3 100644 --- a/docs/testing/mellanox.md +++ b/docs/testing/mellanox.md @@ -70,7 +70,9 @@ ip -n nic4 link set dev enp4s0 up To run commands using `enp3s0` card, use `ip netns exec nic3 `. It is practical to simply call `ip netns exec system1 su ` in a separate terminal. Then it is trivial to listen on this interface to monitor outbound communication for example. -### Single NIC with two ports -This is actually the setup `pytest` suite uses to test on Mellanox. You need to connect the two ports of your NIC. Then the first (the SR-IOV capable) port will be used normally and the second one can be bound to to communicate with dp-service from the outside (host-host communication). +### Two-machine setup +This is actually the setup `pytest` suite uses to test on Mellanox using `--hw` command-line option. You need two *directly connected* machines (without a switch in between). -This requires dp-service running in a mode without a second physical interface. For that to happen you need to compile it with `-Denable_pytest=true` meson flag and not use `--wcmp-frac` command-line option (or any test using port redundancy). +If both Mellanox ports are connected directly to a remote machine, you only need to run `reflector.py` on the remote machine and then start `pytest --hw` on the dpservice-enabled machine. See `reflector.py --help` for command-line options you need to provide (interface names, MACs, optionally virtual service addresses). + +What `reflector.py` does is it takes the underlay traffic and sends it back over the same interface. This way `pytest` can finally listen on PFs and reach the packets sent out from dpservice. To prevent isolation rules from "grabbing" the packet, some packets (i.e. those with a specified MAC) are mangled (the EtherType is changed to `1337`) and then `pytest` changes the type back to IPv6 before processing it further. Thus the packet reflection is transparent to the test suite itself. diff --git a/test/local/dp_service.py b/test/local/dp_service.py index d054b167..3c18a028 100755 --- a/test/local/dp_service.py +++ b/test/local/dp_service.py @@ -24,11 +24,12 @@ def __init__(self, build_path, port_redundancy, fast_flow_timeout, self.hardware = hardware if self.hardware: - raise ValueError("Hardware tests are currently not supported") - if self.port_redundancy: - raise ValueError("Port redundancy is not supported when testing on actual hardware") + # TODO test without pf1-tap + # if self.port_redundancy: + # raise ValueError("Port redundancy is not supported when testing on actual hardware") self.reconfigure_tests(DpService.DP_SERVICE_CONF) else: + # TODO needs testing if offloading: raise ValueError("Offloading is only possible when testing on actual hardware") @@ -113,6 +114,7 @@ def get_vm_tap(self, idx): return iface def reconfigure_tests(self, cfgfile): + pfrepr = "c0pf0" # Rewrite config values to actual hardware values if not os.access(cfgfile, os.R_OK): raise OSError(f"Cannot read {cfgfile} to bind to hardware NIC") @@ -123,9 +125,12 @@ def reconfigure_tests(self, cfgfile): continue key = options[0] value = options[1] - if key == "pf1": - # in hardware, PF0 is actually PF1 as it is used as the monitoring interface connected to the real PF0 + if key == "pf0": PF0.tap = value + elif key == "pf1": + PF1.tap = value + elif key == "pf1-proxy": + PF1.tap = "pf1-tap" elif key == "vf-pattern": # MACs cannot be changed for VFs, use actual values VM1.mac = get_if_hwaddr(f"{value}0") @@ -134,13 +139,16 @@ def reconfigure_tests(self, cfgfile): elif key == "a-pf0": # PCI addresses for VFs are defined by DPDK in this pattern pci = value.split(',')[0] - VM1.pci = f"{pci}_representor_vf0" - VM2.pci = f"{pci}_representor_vf1" - VM3.pci = f"{pci}_representor_vf2" - VM4.pci = f"{pci}_representor_vf3" + elif key == "a-pf1": + # There is a different representor in multiport-eswitch mode and normal mode + pfrepr = "" VM1.tap = self.get_vm_tap(0) VM2.tap = self.get_vm_tap(1) VM3.tap = self.get_vm_tap(2) + VM1.pci = f"{pci}_representor_{pfrepr}vf0" + VM2.pci = f"{pci}_representor_{pfrepr}vf1" + VM3.pci = f"{pci}_representor_{pfrepr}vf2" + VM4.pci = f"{pci}_representor_{pfrepr}vf3" # If run manually: import argparse diff --git a/test/local/helpers.py b/test/local/helpers.py index a9c31889..15c2bc3f 100644 --- a/test/local/helpers.py +++ b/test/local/helpers.py @@ -142,11 +142,20 @@ def validate_checksums(packet): def sniff_packet(iface, lfilter, skip=0): count = skip+1 - pkt_list = sniff(count=count, lfilter=lfilter, iface=iface, timeout=count*sniff_timeout) + # Need to accept 1337 EtherType which is reflected packet used for --hw tests + pkt_list = sniff(count=count, iface=iface, timeout=count*sniff_timeout, + lfilter=lambda pkt: pkt[Ether].type == 0x1337 or lfilter(pkt)) assert len(pkt_list) == count, \ f"No reply on {iface}" pkt = pkt_list[skip] validate_checksums(pkt) + # Reconstruct the original packet for --hw tests + # NOTE: only supports underlay/IPv6 traffic + if pkt[Ether].type == 0x1337: + pkt[Ether].type = 0x86DD + pkt = Ether(pkt.build()) + assert lfilter(pkt), \ + "Reflected packet not of right type" return pkt @@ -191,3 +200,31 @@ def stop_process(process): except subprocess.TimeoutExpired: process.kill() process.wait() + + +def _send_external_icmp_echo(dst_ipv4, ul_ipv6): + icmp_pkt = (Ether(dst=PF0.mac, src=ipv6_multicast_mac, type=0x86DD) / + IPv6(dst=ul_ipv6, src=router_ul_ipv6, nh=4) / + IP(dst=dst_ipv4, src=public_ip) / + ICMP(type=8, id=0x0040)) + delayed_sendp(icmp_pkt, PF0.tap) + +def external_ping(dst_ipv4, ul_ipv6): + threading.Thread(target=_send_external_icmp_echo, args=(dst_ipv4, ul_ipv6)).start() + answer = sniff_packet(PF0.tap, is_icmp_pkt, skip=1) + assert answer[ICMP].code == 0, \ + "Invalid ICMP echo response" + + +def _send_external_icmp_echo6(dst_ipv6, ul_ipv6): + icmp_pkt = (Ether(dst=PF0.mac, src=ipv6_multicast_mac, type=0x86DD) / + IPv6(dst=ul_ipv6, src=router_ul_ipv6, nh=0x29) / + IPv6(dst=dst_ipv6, src=public_ipv6, nh=58) / + ICMPv6EchoRequest()) + delayed_sendp(icmp_pkt, PF0.tap) + +def external_ping6(dst_ipv6, ul_ipv6): + threading.Thread(target=_send_external_icmp_echo6, args=(dst_ipv6, ul_ipv6)).start() + answer = sniff_packet(PF0.tap, is_icmpv6echo_reply_pkt) + assert answer[ICMPv6EchoReply].type == 129, \ + "Invalid ICMPv6 echo response" diff --git a/test/local/reflector.py b/test/local/reflector.py new file mode 100755 index 00000000..74416f44 --- /dev/null +++ b/test/local/reflector.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 + +import argparse +import multiprocessing +import sys +import time +from datetime import datetime +from scapy.all import * + + +def is_ipip_pkt(pkt): + return pkt[IPv6].nh == 4 or pkt[IPv6].nh == 41 + +def is_virtsvc_pkt(pkt): + return pkt[IPv6].src in opts.virtsvc or pkt[IPv6].dst in opts.virtsvc + +# Filter for underlay packets not sent by this script (marked by the Traffic Class field) +def is_ul_pkt(pkt): + return IPv6 in pkt and pkt[IPv6].tc == 0 and (is_ipip_pkt(pkt) or is_virtsvc_pkt(pkt)) + +# Sends packets back over the interface +def sender_loop(q): + try: + while True: + iface, pkt = q.get() + # Give receiver time to enter sniff() + time.sleep(0.1) + sendp(pkt, iface=iface, verbose=opts.verbose > 1) + except KeyboardInterrupt: + return + +# Simply receives all underlay packets and passes them to the sender with appropriate changes +def receiver_loop(q, iface, mac): + if opts.verbose: + print(f"Listening on {iface}, mangling packets from {mac}") + while True: + pkts = sniff(iface=iface, count=1, lfilter=is_ul_pkt) + if len(pkts) != 1: + print(f"Sniffing on {iface} interrupted", file=sys.stderr) + exit(1) + pkt = pkts[0] + if opts.verbose: + summary = f"{pkt.sprintf('%Ether.src% -> %Ether.dst% / %IPv6.src% -> %IPv6.dst%')} / {pkt.summary().split(' / ', 2)[2]}" + print(f"{datetime.now()} {iface}: {summary}") + # Mark the sent-out packet as to not be sniffed by the receiver + pkt[IPv6].tc = 0x0C + # For packets originating from PF, change them so they do not pass directly to dpservice + # But are caught by pytest instead + if pkt[Ether].src == mac: + pkt[Ether].type = 0x1337 + q.put((iface, pkt)) + + +class ReflectAction(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + for value in values: + spec = value.split(',') + if len(spec) != 2: + raise argparse.ArgumentError(self, f"Invalid IFACE,MAC tuple given: '{value}'") + getattr(namespace, self.dest).append(value) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description="Packet reflector for dpservice pytest suite") + parser.add_argument('-v', '--verbose', action='count', default=0, help="more verbose output (use multiple times)") + parser.add_argument('--virtsvc', action='append', default=[], help="virtual service endpoint(s)") + parser.add_argument('reflect', metavar='IFACE,MAC', default=[], nargs='+', action=ReflectAction, help="interface(s) to listen on and *remote* MAC(s) to mangle") + opts = parser.parse_args() + + q = multiprocessing.Queue() + + for spec in opts.reflect: + iface, mac = spec.split(',') + multiprocessing.Process(target=receiver_loop, args=(q, iface, mac)).start() + + sender_loop(q) diff --git a/test/local/test_flows.py b/test/local/test_flows.py index c9083769..962f4646 100644 --- a/test/local/test_flows.py +++ b/test/local/test_flows.py @@ -49,11 +49,11 @@ def test_nat_table_flush(prepare_ipv4, grpc_client, port_redundancy): def send_bounce_pkt_to_pf(ipv6_nat): - bouce_pkt = (Ether(dst=ipv6_multicast_mac, src=PF0.mac, type=0x86DD) / - IPv6(dst=ipv6_nat, src=router_ul_ipv6, nh=4) / - IP(dst=nat_vip, src=public_ip) / - TCP(sport=8989, dport=510)) - delayed_sendp(bouce_pkt, PF0.tap) + bounce_pkt = (Ether(dst=ipv6_multicast_mac, src=PF0.mac, type=0x86DD) / + IPv6(dst=ipv6_nat, src=router_ul_ipv6, nh=4) / + IP(dst=nat_vip, src=public_ip) / + TCP(sport=8989, dport=510)) + delayed_sendp(bounce_pkt, PF0.tap) def test_neighnat_table_flush(prepare_ipv4, grpc_client, port_redundancy): diff --git a/test/local/test_lb.py b/test/local/test_lb.py index de00ea71..6d6dbf2b 100644 --- a/test/local/test_lb.py +++ b/test/local/test_lb.py @@ -5,25 +5,14 @@ from helpers import * -def network_lb_external_icmpv4_ping(lb_ul_ipv6): - icmp_pkt = (Ether(dst=ipv6_multicast_mac, src=PF0.mac, type=0x86DD) / - IPv6(dst=lb_ul_ipv6, src=router_ul_ipv6, nh=4) / - IP(dst=lb_ip, src=public_ip) / - ICMP(type=8, id=0x0040)) - answer = srp1(icmp_pkt, iface=PF0.tap, timeout=sniff_timeout) - validate_checksums(answer) - assert answer and is_icmp_pkt(answer), \ - "No ECHO reply" def test_network_lb_external_icmp_echo(prepare_ipv4, grpc_client): - lb_ul_ipv6 = grpc_client.createlb(lb_name, vni1, lb_ip, "tcp/80") - - network_lb_external_icmpv4_ping(lb_ul_ipv6) - network_lb_external_icmpv4_ping(lb_ul_ipv6) - + external_ping(lb_ip, lb_ul_ipv6) + external_ping(lb_ip, lb_ul_ipv6) grpc_client.dellb(lb_name) + def router_loopback(dst_ipv6, check_ipv4_src, check_ipv4_dst): pkt = sniff_packet(PF0.tap, is_tcp_pkt) assert pkt[IP].dst == check_ipv4_dst, \ @@ -63,12 +52,11 @@ def communicate_vip_lb(vm, lb_ipv6, src_ipv6, src_ipv4, vf_tap, sport): assert vm_reply[TCP].sport == 80, \ f"Invalid server reply port {vm_reply[TCP].sport}" + def test_nat_to_lb_nat(request, prepare_ipv4, grpc_client, port_redundancy): if port_redundancy: pytest.skip("Port redundancy is not supported for NAT <-> LB+NAT test") - if request.config.getoption("--hw"): - pytest.skip("Hardware testing is not supported for NAT <-> LB+NAT test") # Create a VM on VNI1 under a loadbalancer and NAT lb_ul_ipv6 = grpc_client.createlb(lb_name, vni1, lb_ip, "tcp/80") @@ -78,12 +66,9 @@ def test_nat_to_lb_nat(request, prepare_ipv4, grpc_client, port_redundancy): grpc_client.addfwallrule(VM1.name, "fw0-vm1", proto="tcp", dst_port_min=80, dst_port_max=80) # Create another VM on the same VNI behind the same NAT and communicate - VM4.ul_ipv6 = grpc_client.addinterface(VM4.name, VM4.pci, VM4.vni, VM4.ip, VM4.ipv6) - request_ip(VM4) - nat3_ipv6 = grpc_client.addnat(VM4.name, nat_vip, 400, 401) - communicate_vip_lb(VM4, lb_ul_ipv6, nat3_ipv6, nat_vip, VM1.tap, 2400) - grpc_client.delnat(VM4.name) - grpc_client.delinterface(VM4.name) + nat2_ipv6 = grpc_client.addnat(VM2.name, nat_vip, 400, 401) + communicate_vip_lb(VM2, lb_ul_ipv6, nat2_ipv6, nat_vip, VM1.tap, 2400) + grpc_client.delnat(VM2.name) grpc_client.delfwallrule(VM1.name, "fw0-vm1") grpc_client.delnat(VM1.name) @@ -91,6 +76,7 @@ def test_nat_to_lb_nat(request, prepare_ipv4, grpc_client, port_redundancy): grpc_client.dellbprefix(VM1.name, lb_pfx) grpc_client.dellb(lb_name) + def send_bounce_pkt_to_pf(ipv6_lb): bouce_pkt = (Ether(dst=ipv6_multicast_mac, src=PF0.mac, type=0x86DD) / IPv6(dst=ipv6_lb, src=local_ul_ipv6, nh=4) / @@ -113,6 +99,7 @@ def test_external_lb_relay(prepare_ipv4, grpc_client): grpc_client.dellbtarget(lb_name, neigh_ul_ipv6) grpc_client.dellb(lb_name) + def send_bounce_icmp_pkt_to_pf(ipv6_lb): bounce_pkt = (Ether(dst=ipv6_multicast_mac, src=PF0.mac) / IPv6(dst=ipv6_lb, src=local_ul_ipv6, nh=4) / @@ -128,7 +115,6 @@ def test_external_lb_icmp_error_relay(prepare_ipv4, grpc_client): lb_ul_ipv6 = grpc_client.createlb(lb_name, vni1, lb_ip, "tcp/8080") grpc_client.addlbtarget(lb_name, neigh_ul_ipv6) - threading.Thread(target=send_bounce_icmp_pkt_to_pf, args=(lb_ul_ipv6,)).start() pkt = sniff_packet(PF0.tap, is_icmp_pkt, skip=1) @@ -139,25 +125,14 @@ def test_external_lb_icmp_error_relay(prepare_ipv4, grpc_client): grpc_client.dellbtarget(lb_name, neigh_ul_ipv6) grpc_client.dellb(lb_name) -def network_lb_external_icmpv6_ping(lb_ul_ipv6): - icmp_pkt = (Ether(dst=ipv6_multicast_mac, src=PF0.mac, type=0x86DD) / - IPv6(dst=lb_ul_ipv6, src=router_ul_ipv6, nh=0x29) / - IPv6(dst=lb_ip6, src=public_ipv6, nh=58) / - ICMPv6EchoRequest()) - answer = srp1(icmp_pkt, iface=PF0.tap, timeout=sniff_timeout) - validate_checksums(answer) - assert answer and is_icmpv6echo_reply_pkt(answer), \ - "No ECHO reply" def test_network_lb_external_icmpv6_echo(prepare_ipv4, grpc_client): - lb_ul_ipv6 = grpc_client.createlb(lb_name, vni1, lb_ip6, "tcp/443") - - network_lb_external_icmpv6_ping(lb_ul_ipv6) - network_lb_external_icmpv6_ping(lb_ul_ipv6) - + external_ping6(lb_ip6, lb_ul_ipv6) + external_ping6(lb_ip6, lb_ul_ipv6) grpc_client.dellb(lb_name) + def send_bounce_ipv6_pkt_to_pf(ipv6_lb): bounce_pkt = (Ether(dst=ipv6_multicast_mac, src=PF0.mac, type=0x86DD) / IPv6(dst=ipv6_lb, src=local_ul_ipv6, nh=0x29) / @@ -170,7 +145,6 @@ def test_external_lb_relay_ipv6(prepare_ipv4, grpc_client): lb_ul_ipv6 = grpc_client.createlb(lb_name, vni1, lb_ip6, "tcp/8080") grpc_client.addlbtarget(lb_name, neigh_ul_ipv6) - threading.Thread(target=send_bounce_ipv6_pkt_to_pf, args=(lb_ul_ipv6,)).start() pkt = sniff_packet(PF0.tap, is_ipv6_tcp_pkt, skip=1) @@ -179,6 +153,7 @@ def test_external_lb_relay_ipv6(prepare_ipv4, grpc_client): f"Wrong network-lb relayed packet (outer dst ipv6: {dst_ip})" grpc_client.dellb(lb_name) + def test_vip_nat_to_lb_on_another_vni(prepare_ipv4, grpc_client, port_redundancy): if port_redundancy: diff --git a/test/local/test_nat.py b/test/local/test_nat.py index e2724cd0..42d1de01 100644 --- a/test/local/test_nat.py +++ b/test/local/test_nat.py @@ -6,16 +6,10 @@ from helpers import * + def test_network_nat_external_icmp_echo(prepare_ipv4, grpc_client): nat_ul_ipv6 = grpc_client.addnat(VM1.name, nat_vip, nat_local_min_port, nat_local_max_port) - icmp_pkt = (Ether(dst=ipv6_multicast_mac, src=PF0.mac, type=0x86DD) / - IPv6(dst=nat_ul_ipv6, src=router_ul_ipv6, nh=4) / - IP(dst=nat_vip, src=public_ip) / - ICMP(type=8, id=0x0040)) - answer = srp1(icmp_pkt, iface=PF0.tap, timeout=sniff_timeout) - validate_checksums(answer) - assert answer and is_icmp_pkt(answer), \ - "No ECHO reply" + external_ping(nat_vip, nat_ul_ipv6) grpc_client.delnat(VM1.name) def send_bounce_pkt_to_pf(ipv6_nat): diff --git a/test/local/test_telemetry.py b/test/local/test_telemetry.py index 981b50ab..a6c98e7f 100644 --- a/test/local/test_telemetry.py +++ b/test/local/test_telemetry.py @@ -25,6 +25,32 @@ 'rx_errors', 'tx_errors', 'rx_missed_errors', 'rx_mbuf_allocation_errors', 'nat_used_port_count', 'firewall_rule_count', ) +HW_IFACE_STATS = ( + 'rx_broadcast_bytes', 'rx_broadcast_packets', 'tx_broadcast_bytes', 'tx_broadcast_packets', + 'rx_multicast_bytes', 'rx_multicast_packets', 'tx_multicast_bytes', 'tx_multicast_packets', + 'rx_out_of_buffer', + 'rx_phy_bytes', 'rx_phy_crc_errors', 'rx_phy_discard_packets', 'rx_phy_in_range_len_errors', 'rx_phy_packets', 'rx_phy_symbol_errors', + 'tx_phy_bytes', 'tx_phy_discard_packets', 'tx_phy_errors', 'tx_phy_packets', + 'rx_prio0_buf_discard_packets', 'rx_prio0_cong_discard_packets', + 'rx_prio1_buf_discard_packets', 'rx_prio1_cong_discard_packets', + 'rx_prio2_buf_discard_packets', 'rx_prio2_cong_discard_packets', + 'rx_prio3_buf_discard_packets', 'rx_prio3_cong_discard_packets', + 'rx_prio4_buf_discard_packets', 'rx_prio4_cong_discard_packets', + 'rx_prio5_buf_discard_packets', 'rx_prio5_cong_discard_packets', + 'rx_prio6_buf_discard_packets', 'rx_prio6_cong_discard_packets', + 'rx_prio7_buf_discard_packets', 'rx_prio7_cong_discard_packets', + 'rx_unicast_bytes', 'rx_unicast_packets', 'tx_unicast_bytes', 'tx_unicast_packets', + 'rx_vport_bytes', 'rx_vport_packets', 'tx_vport_bytes', 'tx_vport_packets', + 'rx_wqe_errors', + 'tx_pp_clock_queue_errors', 'tx_pp_jitter', 'tx_pp_missed_interrupt_errors', 'tx_pp_rearm_queue_errors', 'tx_pp_sync_lost', 'tx_pp_timestamp_future_errors', 'tx_pp_timestamp_order_errors', 'tx_pp_timestamp_past_errors', 'tx_pp_wander', +) +HW_PF1_IFACE_STATS = ( + 'rx_q1_bytes', 'rx_q1_errors', 'rx_q1_packets', 'tx_q1_bytes', 'tx_q1_packets', + 'rx_q2_bytes', 'rx_q2_errors', 'rx_q2_packets', 'tx_q2_bytes', 'tx_q2_packets', + 'rx_q3_bytes', 'rx_q3_errors', 'rx_q3_packets', 'tx_q3_bytes', 'tx_q3_packets', + 'rx_q4_bytes', 'rx_q4_errors', 'rx_q4_packets', 'tx_q4_bytes', 'tx_q4_packets', + 'rx_q5_bytes', 'rx_q5_errors', 'rx_q5_packets', 'tx_q5_bytes', 'tx_q5_packets', +) HASH_TABLES = ( 'interface_table', 'conntrack_table', 'dnat_table', 'snat_table', @@ -43,7 +69,7 @@ def get_telemetry(request): return response def check_tel_graph(key): - expected_tel_rx_node_count = 6 + expected_tel_rx_node_count = 7 if PF1.tap == "pf1-tap" else 6 tel = get_telemetry(f"/dp_service/graph/{key}") assert tel is not None, \ "Missing graph telemetry" @@ -56,7 +82,7 @@ def check_tel_graph(key): f"Expected {expected_tel_rx_node_count} 'rx-X-0' nodes, found {len(rx_nodes)} in {key} graph telemetry" -def test_telemetry_graph(prepare_ifaces): +def test_telemetry_graph(request, prepare_ifaces): check_tel_graph("obj_count") check_tel_graph("call_count") check_tel_graph("cycle_count") @@ -98,7 +124,7 @@ def test_telemetry_fwall(prepare_ifaces, grpc_client): assert tel == { VM1.name: 0, VM2.name: 0, VM3.name: 0 }, \ "Unexpected firewall rule count" -def test_telemetry_exporter(prepare_ifaces, start_exporter): +def test_telemetry_exporter(request, prepare_ifaces, start_exporter): metrics = urlopen(f"http://localhost:{exporter_port}/metrics").read().decode('utf-8') graph_stats, heap_info, interface_stats, htable_saturation = set(), set(), set(), set() for metric in metrics.splitlines(): @@ -113,15 +139,24 @@ def test_telemetry_exporter(prepare_ifaces, start_exporter): else: assert metric.startswith("#"), \ f"Unknown exported metric '{metric.split('{')[0]}' found" - # meson options (e.g. enable_pf1_proxy) are hard to do in these screipts, so just check manually + # meson options (e.g. enable_pf1_proxy) are hard to do in these scripts, so just check manually graph_nodes = GRAPH_NODES + iface_stats = IFACE_STATS if 'pf1_proxy' in graph_stats: graph_nodes += ('pf1_proxy',) - assert graph_stats == set(graph_nodes) or graph_stats == set(graph_nodes + ('virtsvc',)), \ + if 'virtsvc' in graph_stats: + graph_nodes += ('virtsvc',) + if request.config.getoption("--hw"): + iface_stats += HW_IFACE_STATS + if PF1.tap == "pf1-tap": + graph_nodes += ('rx-6-0',) + if 'rx_q1_bytes' in interface_stats: + iface_stats += HW_PF1_IFACE_STATS + assert graph_stats == set(graph_nodes), \ "Unexpected graph telemetry in exporter output" assert heap_info == set(HEAP_INFO), \ "Unexpected heap info in exporter output" - assert interface_stats == set(IFACE_STATS) or interface_stats == set(IFACE_STATS + ('virtsvc_used_port_count',)), \ + assert interface_stats == set(iface_stats) or interface_stats == set(iface_stats + ('virtsvc_used_port_count',)), \ "Unexpected interface statistics in exporter output" assert htable_saturation == set(HASH_TABLES) or htable_saturation == set(HASH_TABLES + ('virtsvc_table_0', 'virtsvc_table_1')), \ "Unexpected hash table info in exporter output" diff --git a/test/local/test_virtsvc.py b/test/local/test_virtsvc.py index dd539cf1..379b9a5e 100644 --- a/test/local/test_virtsvc.py +++ b/test/local/test_virtsvc.py @@ -47,10 +47,17 @@ def test_virtsvc_udp(request, prepare_ipv4, port_redundancy): if not request.config.getoption("--virtsvc"): pytest.skip("Virtual services not enabled") # port numbers chosen so that they cause the right redirection - for port in [ 12345, 12346, 12347, 12349, 12350 ]: + # (port numbers are part of the hash here, so pf1-proxy changes everything) + pf0_ports = [ 12345, 12346, 12347, 12349, 12350 ] + if request.config.getoption("--hw") and PF1.tap != "pf1-tap": + pf0_ports = [ 12352, 12355, 12360, 12361, 12362 ] + for port in pf0_ports: request_udp(port, PF0.tap) if port_redundancy: - for port in [ 12348, 12351, 12354, 12355, 12357 ]: + pf1_ports = [ 12348, 12351, 12354, 12355, 12357 ] + if request.config.getoption("--hw") and PF1.tap != "pf1-tap": + pf1_ports = [ 12345, 12346, 12347, 12348, 12349 ] + for port in pf1_ports: request_udp(port, PF1.tap) @@ -64,7 +71,8 @@ def test_virtsvc_tcp(request, prepare_ipv4, port_redundancy): if not request.config.getoption("--virtsvc"): pytest.skip("Virtual services not enabled") - tester = TCPTesterVirtsvc(VM1, 12345, PF0, virtsvc_tcp_virtual_ip, virtsvc_tcp_virtual_port, server_pkt_check=tcp_server_virtsvc_pkt_check) + port = 12352 if request.config.getoption("--hw") and PF1.tap != "pf1-tap" else 12345 + tester = TCPTesterVirtsvc(VM1, port, PF0, virtsvc_tcp_virtual_ip, virtsvc_tcp_virtual_port, server_pkt_check=tcp_server_virtsvc_pkt_check) tester.communicate() # port number chosen so that they cause the right redirection From 7ce61c0275b42126425fd0696d59a899b014fe9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jarom=C3=ADr=20Smr=C4=8Dek?= <4plague@gmail.com> Date: Fri, 9 Aug 2024 18:10:50 +0200 Subject: [PATCH 3/3] Support offloading for hardware scapy tests --- docs/testing/mellanox.md | 2 +- test/local/dp_service.py | 4 ---- test/local/tcp_tester.py | 10 +++++++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/testing/mellanox.md b/docs/testing/mellanox.md index ec9d97a3..9f0b0e62 100644 --- a/docs/testing/mellanox.md +++ b/docs/testing/mellanox.md @@ -27,7 +27,7 @@ If you set two VMs like this, they should be able to connect to each other (ping ### Pytest suite As `scapy` cannot directly communicate with VFs, running VMs with two anonymous bridged interfaces are needed. One interface is connected to Mellanox VF using VFIO and the other is a simple TAP device created by KVM. Then dp-service communicates using the VF and test suite uses the TAP device, e.g. ``` --net none -device vfio-pci,host=03:00.02 -device e1000,netdev=tap0 -netdev tap,id=tap0,script=no,downscript=no +qemu-system-x86_64 -enable-kvm -drive media=disk,if=virtio,index=0,file=./kvm/onmetal_tap.qcow2 -m 2G -cpu host -smp 2 -nographic -net none -device vfio-pci,host=03:00.2 -device e1000,netdev=tap2 -netdev tap,id=tap2,script=no,downscript=no ``` Tests have been done using basic Debian 11 installation, but any fresh distro should work, just configure interface like this: diff --git a/test/local/dp_service.py b/test/local/dp_service.py index 3c18a028..048af8cd 100755 --- a/test/local/dp_service.py +++ b/test/local/dp_service.py @@ -24,12 +24,8 @@ def __init__(self, build_path, port_redundancy, fast_flow_timeout, self.hardware = hardware if self.hardware: - # TODO test without pf1-tap - # if self.port_redundancy: - # raise ValueError("Port redundancy is not supported when testing on actual hardware") self.reconfigure_tests(DpService.DP_SERVICE_CONF) else: - # TODO needs testing if offloading: raise ValueError("Offloading is only possible when testing on actual hardware") diff --git a/test/local/tcp_tester.py b/test/local/tcp_tester.py index b41420d9..689fb346 100644 --- a/test/local/tcp_tester.py +++ b/test/local/tcp_tester.py @@ -66,13 +66,17 @@ def reply_tcp(self): self.tcp_receiver_seq += 1 # Application-level reply - if pkt[TCP].payload != None and len(pkt[TCP].payload) > 0: - if pkt[TCP].payload == Raw(_TCPTester.TCP_RESET_REQUEST): + payload = pkt[TCP].payload + if payload != None and len(payload) > 0: + if Padding in payload: + length = len(payload) - len(payload[Padding]) + payload = payload.__class__(raw(payload)[0:length]) + if payload == Raw(_TCPTester.TCP_RESET_REQUEST): reply_pkt = (self.get_server_l3_reply(pkt) / TCP(dport=pkt[TCP].sport, sport=pkt[TCP].dport, seq=self.tcp_receiver_seq, flags="R")) delayed_sendp(reply_pkt, self.server_tap) return - elif pkt[TCP].payload == Raw(_TCPTester.TCP_NORMAL_REQUEST): + elif payload == Raw(_TCPTester.TCP_NORMAL_REQUEST): reply_pkt = (self.get_server_l3_reply(pkt) / TCP(dport=pkt[TCP].sport, sport=pkt[TCP].dport, seq=self.tcp_receiver_seq, flags="") / Raw(_TCPTester.TCP_NORMAL_RESPONSE))