Rework hw pytest

docs/testing/README.md

@@ -31,6 +31,9 @@ When running `pytest` directly in the `test/` directory, only a specific set of
 
 If one should want to instead run your own `dpservice-bin` instance (e.g. for running under a debugger), the `--attach` argument connects to an already running service instead of starting its own (which in turn can be started via a helper `dp_service.py` script). This comes with the caveat of ensuring the right arguments are passed to the service at startup.
 
+#### Pytest on Mellanox
+By default, `pytest` runs using virtual intefaces (TAPs). By providing `--hw` command-line option, it can instead use real NIC based on the contents of `/tmp/dp_service.conf`. For more information [see Mellanox testing guide](mellanox.md#two-machine-setup).
+
 ## Docker
 There is a tester image provided by this repo, simply build it using this repo's `Dockerfile` and use `--target tester`. For fully working images a GitHub PAT is needed: `docker build --secret=id=github_token,src=<path/to/github_token> --target tester .`
 

docs/testing/mellanox.md

@@ -27,7 +27,7 @@ If you set two VMs like this, they should be able to connect to each other (ping
 ### Pytest suite
 As `scapy` cannot directly communicate with VFs, running VMs with two anonymous bridged interfaces are needed. One interface is connected to Mellanox VF using VFIO and the other is a simple TAP device created by KVM. Then dp-service communicates using the VF and test suite uses the TAP device, e.g.
 ```
--net none -device vfio-pci,host=03:00.02 -device e1000,netdev=tap0 -netdev tap,id=tap0,script=no,downscript=no
+qemu-system-x86_64 -enable-kvm -drive media=disk,if=virtio,index=0,file=./kvm/onmetal_tap.qcow2 -m 2G -cpu host -smp 2 -nographic -net none -device vfio-pci,host=03:00.2 -device e1000,netdev=tap2 -netdev tap,id=tap2,script=no,downscript=no
 ```
 
 Tests have been done using basic Debian 11 installation, but any fresh distro should work, just configure interface like this:
@@ -70,7 +70,9 @@ ip -n nic4 link set dev enp4s0 up
 
 To run commands using `enp3s0` card, use `ip netns exec nic3 <command>`. It is practical to simply call `ip netns exec system1 su <username>` in a separate terminal. Then it is trivial to listen on this interface to monitor outbound communication for example.
 
-### Single NIC with two ports
-This is actually the setup `pytest` suite uses to test on Mellanox. You need to connect the two ports of your NIC. Then the first (the SR-IOV capable) port will be used normally and the second one can be bound to to communicate with dp-service from the outside (host-host communication).
+### Two-machine setup
+This is actually the setup `pytest` suite uses to test on Mellanox using `--hw` command-line option. You need two *directly connected* machines (without a switch in between).
 
-This requires dp-service running in a mode without a second physical interface. For that to happen you need to compile it with `-Denable_pytest=true` meson flag and not use `--wcmp-frac` command-line option (or any test using port redundancy).
+If both Mellanox ports are connected directly to a remote machine, you only need to run `reflector.py` on the remote machine and then start `pytest --hw` on the dpservice-enabled machine. See `reflector.py --help` for command-line options you need to provide (interface names, MACs, optionally virtual service addresses).
+
+What `reflector.py` does is it takes the underlay traffic and sends it back over the same interface. This way `pytest` can finally listen on PFs and reach the packets sent out from dpservice. To prevent isolation rules from "grabbing" the packet, some packets (i.e. those with a specified MAC) are mangled (the EtherType is changed to `1337`) and then `pytest` changes the type back to IPv6 before processing it further. Thus the packet reflection is transparent to the test suite itself.

src/dp_port.c

@@ -613,13 +613,8 @@ static int dp_init_port(struct dp_port *port)
 	}
 
 	if (dp_conf_is_offload_enabled()) {
-#ifdef ENABLE_PYTEST
-		if (port->peer_pf_port_id != dp_get_pf1()->port_id)
-#endif
-		{
 		if (DP_FAILED(dp_port_bind_port_hairpins(port)))
 			return DP_ERROR;
-		}
 
 		if (!port->is_pf)
 			if (DP_FAILED(dp_install_vf_init_rte_rules(port)))

src/dp_service.c

@@ -173,17 +173,8 @@ static int init_interfaces(void)
 	if (DP_FAILED(dp_start_port(dp_get_port_by_pf_index(0))))
 		return DP_ERROR;
 
-	// PF1 is always started (can receive from outside) even when not used for Tx
-	// but test suite can use it for monitoring in some setups, so do not bind it then
-#ifdef ENABLE_PYTEST
-#ifndef ENABLE_PF1_PROXY
-	if (dp_conf_is_wcmp_enabled())
-#endif
-#endif
-	{
 	if (DP_FAILED(dp_start_port(dp_get_port_by_pf_index(1))))
 		return DP_ERROR;
-	}
 
 #ifdef ENABLE_PF1_PROXY
 	if (DP_FAILED(dp_start_pf_proxy_tap_port()))

test/local/dp_service.py

@@ -24,8 +24,6 @@ def __init__(self, build_path, port_redundancy, fast_flow_timeout,
 		self.hardware = hardware
 
 		if self.hardware:
-			if self.port_redundancy:
-				raise ValueError("Port redundancy is not supported when testing on actual hardware")
 			self.reconfigure_tests(DpService.DP_SERVICE_CONF)
 		else:
 			if offloading:
@@ -112,6 +110,7 @@ def get_vm_tap(self, idx):
 		return iface
 
 	def reconfigure_tests(self, cfgfile):
+		pfrepr = "c0pf0"
 		# Rewrite config values to actual hardware values
 		if not os.access(cfgfile, os.R_OK):
 			raise OSError(f"Cannot read {cfgfile} to bind to hardware NIC")
@@ -122,9 +121,12 @@ def reconfigure_tests(self, cfgfile):
 					continue
 				key = options[0]
 				value = options[1]
-				if key == "pf1":
-					# in hardware, PF0 is actually PF1 as it is used as the monitoring interface connected to the real PF0
+				if key == "pf0":
 					PF0.tap = value
+				elif key == "pf1":
+					PF1.tap = value
+				elif key == "pf1-proxy":
+					PF1.tap = "pf1-tap"
 				elif key == "vf-pattern":
 					# MACs cannot be changed for VFs, use actual values
 					VM1.mac = get_if_hwaddr(f"{value}0")
@@ -133,13 +135,16 @@ def reconfigure_tests(self, cfgfile):
 				elif key == "a-pf0":
 					# PCI addresses for VFs are defined by DPDK in this pattern
 					pci = value.split(',')[0]
-					VM1.pci = f"{pci}_representor_vf0"
-					VM2.pci = f"{pci}_representor_vf1"
-					VM3.pci = f"{pci}_representor_vf2"
-					VM4.pci = f"{pci}_representor_vf3"
+				elif key == "a-pf1":
+					# There is a different representor in multiport-eswitch mode and normal mode
+					pfrepr = ""
 		VM1.tap = self.get_vm_tap(0)
 		VM2.tap = self.get_vm_tap(1)
 		VM3.tap = self.get_vm_tap(2)
+		VM1.pci = f"{pci}_representor_{pfrepr}vf0"
+		VM2.pci = f"{pci}_representor_{pfrepr}vf1"
+		VM3.pci = f"{pci}_representor_{pfrepr}vf2"
+		VM4.pci = f"{pci}_representor_{pfrepr}vf3"
 
 # If run manually:
 import argparse

test/local/helpers.py

@@ -142,11 +142,20 @@ def validate_checksums(packet):
 
 def sniff_packet(iface, lfilter, skip=0):
 	count = skip+1
-	pkt_list = sniff(count=count, lfilter=lfilter, iface=iface, timeout=count*sniff_timeout)
+	# Need to accept 1337 EtherType which is reflected packet used for --hw tests
+	pkt_list = sniff(count=count, iface=iface, timeout=count*sniff_timeout,
+					 lfilter=lambda pkt: pkt[Ether].type == 0x1337 or lfilter(pkt))
 	assert len(pkt_list) == count, \
 		f"No reply on {iface}"
 	pkt = pkt_list[skip]
 	validate_checksums(pkt)
+	# Reconstruct the original packet for --hw tests
+	# NOTE: only supports underlay/IPv6 traffic
+	if pkt[Ether].type == 0x1337:
+		pkt[Ether].type = 0x86DD
+		pkt = Ether(pkt.build())
+		assert lfilter(pkt), \
+			"Reflected packet not of right type"
 	return pkt
 
 
@@ -191,3 +200,31 @@ def stop_process(process):
 	except subprocess.TimeoutExpired:
 		process.kill()
 		process.wait()
+
+
+def _send_external_icmp_echo(dst_ipv4, ul_ipv6):
+	icmp_pkt = (Ether(dst=PF0.mac, src=ipv6_multicast_mac, type=0x86DD) /
+			    IPv6(dst=ul_ipv6, src=router_ul_ipv6, nh=4) /
+			    IP(dst=dst_ipv4, src=public_ip) /
+			    ICMP(type=8, id=0x0040))
+	delayed_sendp(icmp_pkt, PF0.tap)
+
+def external_ping(dst_ipv4, ul_ipv6):
+	threading.Thread(target=_send_external_icmp_echo, args=(dst_ipv4, ul_ipv6)).start()
+	answer = sniff_packet(PF0.tap, is_icmp_pkt, skip=1)
+	assert answer[ICMP].code == 0, \
+		"Invalid ICMP echo response"
+
+
+def _send_external_icmp_echo6(dst_ipv6, ul_ipv6):
+	icmp_pkt = (Ether(dst=PF0.mac, src=ipv6_multicast_mac, type=0x86DD) /
+				IPv6(dst=ul_ipv6, src=router_ul_ipv6, nh=0x29) /
+				IPv6(dst=dst_ipv6, src=public_ipv6, nh=58) /
+				ICMPv6EchoRequest())
+	delayed_sendp(icmp_pkt, PF0.tap)
+
+def external_ping6(dst_ipv6, ul_ipv6):
+	threading.Thread(target=_send_external_icmp_echo6, args=(dst_ipv6, ul_ipv6)).start()
+	answer = sniff_packet(PF0.tap, is_icmpv6echo_reply_pkt)
+	assert answer[ICMPv6EchoReply].type == 129, \
+		"Invalid ICMPv6 echo response"

test/local/reflector.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+
+import argparse
+import multiprocessing
+import sys
+import time
+from datetime import datetime
+from scapy.all import *
+
+
+def is_ipip_pkt(pkt):
+	return pkt[IPv6].nh == 4 or pkt[IPv6].nh == 41
+
+def is_virtsvc_pkt(pkt):
+	return pkt[IPv6].src in opts.virtsvc or pkt[IPv6].dst in opts.virtsvc
+
+# Filter for underlay packets not sent by this script (marked by the Traffic Class field)
+def is_ul_pkt(pkt):
+	return IPv6 in pkt and pkt[IPv6].tc == 0 and (is_ipip_pkt(pkt) or is_virtsvc_pkt(pkt))
+
+# Sends packets back over the interface
+def sender_loop(q):
+	try:
+		while True:
+			iface, pkt = q.get()
+			# Give receiver time to enter sniff()
+			time.sleep(0.1)
+			sendp(pkt, iface=iface, verbose=opts.verbose > 1)
+	except KeyboardInterrupt:
+		return
+
+# Simply receives all underlay packets and passes them to the sender with appropriate changes
+def receiver_loop(q, iface, mac):
+	if opts.verbose:
+		print(f"Listening on {iface}, mangling packets from {mac}")
+	while True:
+		pkts = sniff(iface=iface, count=1, lfilter=is_ul_pkt)
+		if len(pkts) != 1:
+			print(f"Sniffing on {iface} interrupted", file=sys.stderr)
+			exit(1)
+		pkt = pkts[0]
+		if opts.verbose:
+			summary = f"{pkt.sprintf('%Ether.src% -> %Ether.dst% / %IPv6.src% -> %IPv6.dst%')} / {pkt.summary().split(' / ', 2)[2]}"
+			print(f"{datetime.now()} {iface}: {summary}")
+		# Mark the sent-out packet as to not be sniffed by the receiver
+		pkt[IPv6].tc = 0x0C
+		# For packets originating from PF, change them so they do not pass directly to dpservice
+		# But are caught by pytest instead
+		if pkt[Ether].src == mac:
+			pkt[Ether].type = 0x1337
+		q.put((iface, pkt))
+
+
+class ReflectAction(argparse.Action):
+	def __call__(self, parser, namespace, values, option_string=None):
+		for value in values:
+			spec = value.split(',')
+			if len(spec) != 2:
+				raise argparse.ArgumentError(self, f"Invalid IFACE,MAC tuple given: '{value}'")
+			getattr(namespace, self.dest).append(value)
+
+if __name__ == '__main__':
+	parser = argparse.ArgumentParser(description="Packet reflector for dpservice pytest suite")
+	parser.add_argument('-v', '--verbose', action='count', default=0, help="more verbose output (use multiple times)")
+	parser.add_argument('--virtsvc', action='append', default=[], help="virtual service endpoint(s)")
+	parser.add_argument('reflect', metavar='IFACE,MAC', default=[], nargs='+', action=ReflectAction, help="interface(s) to listen on and *remote* MAC(s) to mangle")
+	opts = parser.parse_args()
+
+	q = multiprocessing.Queue()
+
+	for spec in opts.reflect:
+		iface, mac = spec.split(',')
+		multiprocessing.Process(target=receiver_loop, args=(q, iface, mac)).start()
+
+	sender_loop(q)

test/local/tcp_tester.py

@@ -66,13 +66,17 @@ def reply_tcp(self):
 			self.tcp_receiver_seq += 1
 
 		# Application-level reply
-		if pkt[TCP].payload != None and len(pkt[TCP].payload) > 0:
-			if pkt[TCP].payload == Raw(_TCPTester.TCP_RESET_REQUEST):
+		payload = pkt[TCP].payload
+		if payload != None and len(payload) > 0:
+			if Padding in payload:
+				length = len(payload) - len(payload[Padding])
+				payload = payload.__class__(raw(payload)[0:length])
+			if payload == Raw(_TCPTester.TCP_RESET_REQUEST):
 				reply_pkt = (self.get_server_l3_reply(pkt) /
 							 TCP(dport=pkt[TCP].sport, sport=pkt[TCP].dport, seq=self.tcp_receiver_seq, flags="R"))
 				delayed_sendp(reply_pkt, self.server_tap)
 				return
-			elif pkt[TCP].payload == Raw(_TCPTester.TCP_NORMAL_REQUEST):
+			elif payload == Raw(_TCPTester.TCP_NORMAL_REQUEST):
 				reply_pkt = (self.get_server_l3_reply(pkt) /
 							 TCP(dport=pkt[TCP].sport, sport=pkt[TCP].dport, seq=self.tcp_receiver_seq, flags="") /
 							 Raw(_TCPTester.TCP_NORMAL_RESPONSE))

test/local/test_flows.py

@@ -49,11 +49,11 @@ def test_nat_table_flush(prepare_ipv4, grpc_client, port_redundancy):
 
 
 def send_bounce_pkt_to_pf(ipv6_nat):
-	bouce_pkt = (Ether(dst=ipv6_multicast_mac, src=PF0.mac, type=0x86DD) /
-				 IPv6(dst=ipv6_nat, src=router_ul_ipv6, nh=4) /
-				 IP(dst=nat_vip, src=public_ip) /
-				 TCP(sport=8989, dport=510))
-	delayed_sendp(bouce_pkt, PF0.tap)
+	bounce_pkt = (Ether(dst=ipv6_multicast_mac, src=PF0.mac, type=0x86DD) /
+				  IPv6(dst=ipv6_nat, src=router_ul_ipv6, nh=4) /
+				  IP(dst=nat_vip, src=public_ip) /
+				  TCP(sport=8989, dport=510))
+	delayed_sendp(bounce_pkt, PF0.tap)
 
 def test_neighnat_table_flush(prepare_ipv4, grpc_client, port_redundancy):