Fix various critical bugs on Windows (#4467)

* Cleanup windows native mode and fix Windows tests.

Note: this module should be considered a last chance only, as it comes
with MANY limitations.

* Remove allow_failures on appveyor

* Improve select_objects thanks to WSAEventSelect

* Restore support for legacy Npcap adapter (<0.9983)

* AppVeyor: upgrade tested Python version

* Fix tox breaking AGAIN

* Disable native TLS1.3 for the ancient Windows used by AppVeyor

* Improvements to SSLStreamSocket on Windows

* Disable unstable windows tests

* Disable broken DoIP tests on Windows

* Minor HTTP bugfix

.appveyor.yml

@@ -7,26 +7,23 @@ environment:
   # Python versions that will be tested
   # Note: it defines variables that can be used later
   matrix:
-    - PYTHON: "C:\\Python37-x64"
-      PYTHON_VERSION: "3.7.x"
+    - PYTHON: "C:\\Python312-x64"
+      PYTHON_VERSION: "3.12.x"
       PYTHON_ARCH: "64"
-      TOXENV: "py37-windows"
+      TOXENV: "py312-windows"
       UT_FLAGS: "-K scanner"
-    - PYTHON: "C:\\Python37-x64"
-      PYTHON_VERSION: "3.7.x"
+    - PYTHON: "C:\\Python312-x64"
+      PYTHON_VERSION: "3.12.x"
       PYTHON_ARCH: "64"
-      TOXENV: "py37-windows"
+      TOXENV: "py312-windows"
       UT_FLAGS: "-k scanner"
 
-# allow scanner builds to fail
-matrix:
-  allow_failures:
-    - UT_FLAGS: "-k scanner"
-
 # There is no build phase for Scapy
 build: off
 
 install:
+  # Log some debug info
+  - ver
   # Install the npcap, windump and wireshark suites
   - ps: .\.config\appveyor\InstallNpcap.ps1
   - ps: .\.config\appveyor\InstallWindumpNpcap.ps1
@@ -43,7 +40,7 @@ test_script:
   # Set environment variables
   - set PYTHONPATH=%APPVEYOR_BUILD_FOLDER%
   - set PATH=%APPVEYOR_BUILD_FOLDER%;C:\Program Files\Wireshark\;C:\Program Files\Windump\;%PATH%
-  - set TOX_PARALLEL_NO_SPINNER=1
+  # - set TOX_PARALLEL_NO_SPINNER=1
 
   # Main unit tests
   - "%PYTHON%\\python -m tox -- %UT_FLAGS%"

.config/ci/test.sh

@@ -117,7 +117,8 @@ then
 fi
 
 # Launch Scapy unit tests
-TOX_PARALLEL_NO_SPINNER=1 tox -- ${UT_FLAGS} || exit 1
+# export TOX_PARALLEL_NO_SPINNER=1
+tox -- ${UT_FLAGS} || exit 1
 
 # Stop if NO_BASH_TESTS is set
 if [ ! -z "$SIMPLE_TESTS" ]

scapy/arch/windows/__init__.py

@@ -71,6 +71,7 @@
 
 # Detection happens after libpcap import (NPcap detection)
 NPCAP_LOOPBACK_NAME = r"\Device\NPF_Loopback"
+NPCAP_LOOPBACK_NAME_LEGACY = "Npcap Loopback Adapter"  # before npcap 0.9983
 if conf.use_npcap:
     conf.loopback_name = NPCAP_LOOPBACK_NAME
 else:
@@ -342,7 +343,7 @@ def update(self, data):
 
         try:
             # Npcap loopback interface
-            if conf.use_npcap and self.network_name == NPCAP_LOOPBACK_NAME:
+            if conf.use_npcap and self.network_name == conf.loopback_name:
                 # https://nmap.org/npcap/guide/npcap-devguide.html
                 data["mac"] = "00:00:00:00:00:00"
                 data["ip"] = "127.0.0.1"
@@ -602,14 +603,20 @@ def load(self, NetworkInterface_Win=NetworkInterface_Win):
             # Try a restart
             WindowsInterfacesProvider._pcap_check()
 
+        legacy_npcap_guid = None
         windows_interfaces = dict()
         for i in get_windows_if_list():
-            # Detect Loopback interface
-            if "Loopback" in i['name']:
-                i['name'] = conf.loopback_name
+            # Only consider interfaces with a GUID
             if i['guid']:
-                if conf.use_npcap and i['name'] == conf.loopback_name:
-                    i['guid'] = NPCAP_LOOPBACK_NAME
+                if conf.use_npcap:
+                    # Detect the legacy Loopback interface
+                    if i['name'] == NPCAP_LOOPBACK_NAME_LEGACY:
+                        # Legacy Npcap (<0.9983)
+                        legacy_npcap_guid = i['guid']
+                    elif "Loopback" in i['name']:
+                        # Newer Npcap
+                        i['guid'] = conf.loopback_name
+                # Map interface
                 windows_interfaces[i['guid']] = i
 
         def iterinterfaces() -> Iterator[
@@ -621,12 +628,16 @@ def iterinterfaces() -> Iterator[
                 for netw, if_data in conf.cache_pcapiflist.items():
                     name, ips, flags, _ = if_data
                     guid = _pcapname_to_guid(netw)
+                    if guid == legacy_npcap_guid:
+                        # Legacy Npcap detected !
+                        conf.loopback_name = netw
                     data = windows_interfaces.get(guid, None)
                     yield netw, name, ips, flags, guid, data
             else:
                 # We don't have a libpcap provider: only use Windows data
                 for guid, data in windows_interfaces.items():
-                    yield guid, None, [], 0, guid, data
+                    netw = r'\Device\NPF_' + guid if guid[0] != '\\' else guid
+                    yield netw, None, [], 0, guid, data
 
         index = 0
         for netw, name, ips, flags, guid, data in iterinterfaces():
@@ -1021,7 +1032,7 @@ def __init__(self, *args, **kargs):
         # type: (*Any, **Any) -> None
         raise RuntimeError(
             "Sniffing and sending packets is not available at layer 2: "
-            "winpcap is not installed. You may use conf.L3socket or"
+            "winpcap is not installed. You may use conf.L3socket or "
             "conf.L3socket6 to access layer 3"
         )
 

scapy/arch/windows/native.py

@@ -6,49 +6,23 @@
 """
 Native Microsoft Windows sockets (L3 only)
 
-## Notice: ICMP packets
+This uses Raw Sockets from winsock
+https://learn.microsoft.com/en-us/windows/win32/winsock/tcp-ip-raw-sockets-2
 
-DISCLAIMER: Please use Npcap/Winpcap to send/receive ICMP. It is going to work.
-Below is some additional information, mainly implemented in a testing purpose.
+.. note::
 
-When in native mode, everything goes through the Windows kernel.
-This firstly requires that the Firewall is open. Be sure it allows ICMPv4/6
-packets in and out.
-Windows may drop packets that it finds wrong. for instance, answers to
-ICMP packets with id=0 or seq=0 may be dropped. It means that sent packets
-should (most of the time) be perfectly built.
-
-A perfectly built ICMP req packet on Windows means that its id is 1, its
-checksum (IP and ICMP) are correctly built, but also that its seq number is
-in the "allowed range".
-    In fact, every time an ICMP packet is sent on Windows, a global sequence
-number is increased, which is only reset at boot time. The seq number of the
-received ICMP packet must be in the range [current, current + 3] to be valid,
-and received by the socket. The current number is quite hard to get, thus we
-provide in this module the get_actual_icmp_seq() function.
-
-Example:
-    >>> conf.use_pcap = False
-    >>> a = conf.L3socket()
-    # This will (most likely) work:
-    >>> current = get_current_icmp_seq()
-    >>> a.sr(IP(dst="www.google.com", ttl=128)/ICMP(id=1, seq=current))
-    # This won't:
-    >>> a.sr(IP(dst="www.google.com", ttl=128)/ICMP())
-
-PS: on computers where the firewall isn't open, Windows temporarily opens it
-when using the `ping` util from cmd.exe. One can first call a ping on cmd,
-then do custom calls through the socket using get_current_icmp_seq(). See
-the tests (windows.uts) for an example.
+    Don't use this module.
+    It is a proof of concept, and a worse-case-scenario failover, but you should
+    consider that raw sockets on Windows don't work and install Npcap to avoid using
+    it at all cost.
 """
+
 import io
-import os
 import socket
-import subprocess
+import struct
 import time
 
 from scapy.automaton import select_objects
-from scapy.arch.windows.structures import GetIcmpStatistics
 from scapy.compat import raw
 from scapy.config import conf
 from scapy.data import MTU
@@ -70,14 +44,31 @@
 
 
 class L3WinSocket(SuperSocket):
+    """
+    A L3 raw socket implementation native to Windows.
+
+    Official "Windows Limitations" from MSDN:
+        - TCP data cannot be sent over raw sockets.
+        - UDP datagrams with an invalid source address cannot be sent over raw sockets.
+        - For IPv6 (address family of AF_INET6), an application receives everything
+          after the last IPv6 header in each received datagram [...]. The application
+          does not receive any IPv6 headers using a raw socket.
+
+    Unofficial limitations:
+        - Turns out we actually don't see any incoming TCP data, only the outgoing.
+          We do properly see UDP, ICMP, etc. both ways though.
+        - To match IPv6 responses, one must use `conf.checkIPaddr = False` as we can't
+          get the real destination.
+
+    **To overcome those limitations, install Npcap.**
+    """
     desc = "a native Layer 3 (IPv4) raw socket under Windows"
     nonblocking_socket = True
     __selectable_force_select__ = True  # see automaton.py
-    __slots__ = ["promisc", "cls", "ipv6", "proto"]
+    __slots__ = ["promisc", "cls", "ipv6"]
 
     def __init__(self,
                  iface=None,  # type: Optional[_GlobInterfaceType]
-                 proto=None,  # type: Optional[int]
                  ttl=128,  # type: int
                  ipv6=False,  # type: bool
                  promisc=True,  # type: bool
@@ -89,62 +80,39 @@ def __init__(self,
         for kwarg in kwargs:
             log_runtime.warning("Dropping unsupported option: %s" % kwarg)
         self.iface = iface and resolve_iface(iface) or conf.iface
+        if not self.iface.is_valid():
+            log_runtime.warning("Interface is invalid. This will fail.")
         af = socket.AF_INET6 if ipv6 else socket.AF_INET
         self.ipv6 = ipv6
-        # Proto and cls
-        if proto is None:
-            if self.ipv6:
-                # On IPv6, the header isn't returned with recvfrom().
-                # We don't want to guess if it's TCP, UDP or SCTP.. so ask for proto
-                # (This would be fixable if Python supported recvmsg() on Windows)
-                log_runtime.warning(
-                    "Due to restrictions, 'proto' must be provided when "
-                    "opening raw IPv6 sockets. Defaulting to socket.IPPROTO_UDP"
-                )
-                self.proto = socket.IPPROTO_UDP
-            else:
-                self.proto = socket.IPPROTO_IP
-        elif self.ipv6 and proto == socket.IPPROTO_TCP:
-            # Ah, sadly this isn't supported either.
-            log_runtime.warning(
-                "Be careful, socket.IPPROTO_TCP doesn't work in raw sockets on "
-                "Windows, so this is equivalent to socket.IPPROTO_IP."
-            )
-            self.proto = socket.IPPROTO_IP
-        else:
-            self.proto = proto
         self.cls = IPv6 if ipv6 else IP
         # Promisc
         if promisc is None:
             promisc = conf.sniff_promisc
         self.promisc = promisc
         # Notes:
-        # - IPPROTO_RAW only works to send packets.
+        # - IPPROTO_RAW is broken. We don't use it.
         # - IPPROTO_IPV6 exists in MSDN docs, but using it will result in
         # no packets being received. Same for its options (IPV6_HDRINCL...)
         # However, using IPPROTO_IP with AF_INET6 will still receive
         # the IPv6 packets
         try:
             # Listening on AF_INET6 IPPROTO_IPV6 is broken. Use IPPROTO_IP
-            self.ins = socket.socket(af,
-                                     socket.SOCK_RAW,
-                                     socket.IPPROTO_IP)
-            self.outs = socket.socket(af,
-                                      socket.SOCK_RAW,
-                                      socket.IPPROTO_RAW)
+            self.outs = self.ins = socket.socket(
+                af,
+                socket.SOCK_RAW,
+                socket.IPPROTO_IP,
+            )
         except OSError as e:
             if e.errno == 13:
                 raise OSError("Windows native L3 Raw sockets are only "
                               "usable as administrator ! "
                               "Please install Npcap to workaround !")
             raise
         self.ins.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-        self.outs.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
         self.ins.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 2**30)
         self.outs.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 2**30)
         # set TTL
         self.ins.setsockopt(socket.IPPROTO_IP, socket.IP_TTL, ttl)
-        self.outs.setsockopt(socket.IPPROTO_IP, socket.IP_TTL, ttl)
         # Get as much data as possible: reduce what is cropped
         if ipv6:
             # IPV6_HDRINCL is broken. Use IP_HDRINCL even on IPv6
@@ -159,7 +127,6 @@ def __init__(self,
         else:
             # IOCTL Include IP headers
             self.ins.setsockopt(socket.IPPROTO_IP, socket.IP_HDRINCL, 1)
-            self.outs.setsockopt(socket.IPPROTO_IP, socket.IP_HDRINCL, 1)
             try:  # Not Windows XP
                 self.ins.setsockopt(socket.IPPROTO_IP,
                                     socket.IP_RECVDSTADDR, 1)
@@ -193,6 +160,12 @@ def send(self, x):
         if self.cls not in x:
             raise Scapy_Exception("L3WinSocket can only send IP/IPv6 packets !"
                                   " Install Npcap/Winpcap to send more")
+        from scapy.layers.inet import TCP
+        if TCP in x:
+            raise Scapy_Exception(
+                "'TCP data cannot be sent over raw socket': "
+                "https://learn.microsoft.com/en-us/windows/win32/winsock/tcp-ip-raw-sockets-2"  # noqa: E501
+            )
         if not self.outs:
             raise Scapy_Exception("Socket not created")
         dst_ip = str(x[self.cls].dst)
@@ -225,14 +198,20 @@ def recv_raw(self, x=MTU):
             # AF_INET6 does not return the IPv6 header. Let's build it
             # (host, port, flowinfo, scopeid)
             host, _, flowinfo, _ = address
+            # We have to guess what the proto is. Ugly heuristics ahead :(
+            # Waiting for https://github.com/python/cpython/issues/80398
+            if len(data) > 6 and struct.unpack("!H", data[4:6])[0] == len(data):
+                proto = socket.IPPROTO_UDP
+            elif data and data[0] in range(128, 138):  # ugh
+                proto = socket.IPPROTO_ICMPV6
+            else:
+                proto = socket.IPPROTO_TCP
             header = raw(
                 IPv6(
                     src=host,
                     dst="::",
                     fl=flowinfo,
-                    # when IPPROTO_IP (0) is selected, we have no idea what's nh,
-                    # so set an invalid value.
-                    nh=self.proto or 0xFF,
+                    nh=proto or 0xFF,
                     plen=len(data)
                 )
             )
@@ -262,22 +241,3 @@ def __init__(self, **kwargs):
             ipv6=True,
             **kwargs,
         )
-
-
-def open_icmp_firewall(host):
-    # type: (str) -> int
-    """Temporarily open the ICMP firewall. Tricks Windows into allowing
-    ICMP packets for a short period of time (~ 1 minute)"""
-    # We call ping with a timeout of 1ms: will return instantly
-    with open(os.devnull, 'wb') as DEVNULL:
-        return subprocess.Popen("ping -4 -w 1 -n 1 %s" % host,
-                                shell=True,
-                                stdout=DEVNULL,
-                                stderr=DEVNULL).wait()
-
-
-def get_current_icmp_seq():
-    # type: () -> int
-    """See help(scapy.arch.windows.native) for more information.
-    Returns the current ICMP seq number."""
-    return GetIcmpStatistics()['stats']['icmpOutStats']['dwEchos']