Fix bugs in HTTP implementation (#4438)

* Small bug fixes when parsing HTTP

- new auto_chunk parameter. add handling of build
- fix TCPSession not properly handling empty packets in app=True mode
- fix bug where StringBuffer would add an empty byte \x00 when passed an
  empty string

* Add chunk test

scapy/layers/http.py

@@ -32,6 +32,12 @@
 
     >>> conf.contribs["http"]["auto_compression"] = False
 
+(Defaults to True)
+
+You can also turn auto-chunking/dechunking off with::
+
+    >>> conf.contribs["http"]["auto_chunk"] = False
+
 (Defaults to True)
 """
 
@@ -92,6 +98,7 @@
 if "http" not in conf.contribs:
     conf.contribs["http"] = {}
     conf.contribs["http"]["auto_compression"] = True
+    conf.contribs["http"]["auto_chunk"] = True
 
 # https://en.wikipedia.org/wiki/List_of_HTTP_header_fields
 
@@ -302,11 +309,9 @@ def hashret(self):
 
     def post_dissect(self, s):
         self._original_len = len(s)
-        if not conf.contribs["http"]["auto_compression"]:
-            return s
         encodings = self._get_encodings()
         # Un-chunkify
-        if "chunked" in encodings:
+        if conf.contribs["http"]["auto_chunk"] and "chunked" in encodings:
             data = b""
             while s:
                 length, _, body = s.partition(b"\r\n")
@@ -324,6 +329,8 @@ def post_dissect(self, s):
                     data += load
             if not s:
                 s = data
+        if not conf.contribs["http"]["auto_compression"]:
+            return s
         # Decompress
         try:
             if "deflate" in encodings:
@@ -366,39 +373,42 @@ def post_dissect(self, s):
         return s
 
     def post_build(self, pkt, pay):
-        if not conf.contribs["http"]["auto_compression"]:
-            return pkt + pay
         encodings = self._get_encodings()
-        # Compress
-        if "deflate" in encodings:
-            import zlib
-            pay = zlib.compress(pay)
-        elif "gzip" in encodings:
-            pay = gzip.compress(pay)
-        elif "compress" in encodings:
-            if _is_lzw_available:
-                pay = lzw.compress(pay)
-            else:
-                log_loading.info(
-                    "Can't import lzw. compress compression "
-                    "will be ignored !"
-                )
-        elif "br" in encodings:
-            if _is_brotli_available:
-                pay = brotli.compress(pay)
-            else:
-                log_loading.info(
-                    "Can't import brotli. brotli compression will "
-                    "be ignored !"
-                )
-        elif "zstd" in encodings:
-            if _is_zstd_available:
-                pay = zstandard.ZstdCompressor().compress(pay)
-            else:
-                log_loading.info(
-                    "Can't import zstandard. zstd compression will "
-                    "be ignored !"
-                )
+        if conf.contribs["http"]["auto_compression"]:
+            # Compress
+            if "deflate" in encodings:
+                import zlib
+                pay = zlib.compress(pay)
+            elif "gzip" in encodings:
+                pay = gzip.compress(pay)
+            elif "compress" in encodings:
+                if _is_lzw_available:
+                    pay = lzw.compress(pay)
+                else:
+                    log_loading.info(
+                        "Can't import lzw. compress compression "
+                        "will be ignored !"
+                    )
+            elif "br" in encodings:
+                if _is_brotli_available:
+                    pay = brotli.compress(pay)
+                else:
+                    log_loading.info(
+                        "Can't import brotli. brotli compression will "
+                        "be ignored !"
+                    )
+            elif "zstd" in encodings:
+                if _is_zstd_available:
+                    pay = zstandard.ZstdCompressor().compress(pay)
+                else:
+                    log_loading.info(
+                        "Can't import zstandard. zstd compression will "
+                        "be ignored !"
+                    )
+        # Chunkify
+        if conf.contribs["http"]["auto_chunk"] and "chunked" in encodings:
+            # Dumb: 1 single chunk.
+            pay = (b"%X" % len(pay)) + b"\r\n" + pay + b"\r\n0\r\n\r\n"
         return pkt + pay
 
     def self_build(self, **kwargs):
@@ -643,8 +653,13 @@ def tcp_reassemble(cls, data, metadata, _):
             # Packets may have a Content-Length we must honnor
             length = http_packet.Content_Length
             # Heuristic to try and detect instant HEAD responses, as those include a
-            # Content-Length that must not be honored.
-            if is_response and data.endswith(b"\r\n\r\n"):
+            # Content-Length that must not be honored. This is a bit crappy, and assumes
+            # that a 'HEAD' will never include an Encoding...
+            if (
+                is_response and
+                data.endswith(b"\r\n\r\n") and
+                not http_packet[HTTPResponse]._get_encodings()
+            ):
                 detect_end = lambda _: True
             elif length is not None:
                 # The packet provides a Content-Length attribute: let's

scapy/sessions.py

@@ -104,6 +104,8 @@ def __init__(self):
         self.incomplete = []  # type: List[Tuple[int, int]]
 
     def append(self, data: bytes, seq: Optional[int] = None) -> None:
+        if not data:
+            return
         data_len = len(data)
         if seq is None:
             seq = self.content_len
@@ -136,7 +138,7 @@ def full(self):
         # type: () -> bool
         # Should only be true when all missing data was filled up,
         # (or there never was missing data)
-        return True  # XXX
+        return bool(self)
 
     def clear(self):
         # type: () -> None
@@ -275,11 +277,12 @@ def process(self,
                 self.metadata["tcp_reassemble"] = tcp_reassemble = streamcls(cls)
             else:
                 return None
-            packet = tcp_reassemble(
-                bytes(self.data),
-                self.metadata,
-                self.session,
-            )
+            if self.data.full():
+                packet = tcp_reassemble(
+                    bytes(self.data),
+                    self.metadata,
+                    self.session,
+                )
             if packet:
                 padding = self._strip_padding(packet)
                 if padding:

test/scapy/layers/http.uts

@@ -85,6 +85,11 @@ assert a[5].Expires == b'Mon, 01 Apr 2024 22:25:38 GMT'
 assert a[5].Reason_Phrase == b'Moved Permanently'
 assert a[5].X_Frame_Options == b"SAMEORIGIN"
 
+= HTTP build with 'chunked' content type
+
+pkt = HTTP()/HTTPResponse(Content_Encoding="chunked", Date=b'Sat, 22 Jun 2024 10:00:00 GMT')/(b"A" * 100)
+assert bytes(pkt) == b'HTTP/1.1 200 OK\r\nContent-Encoding: chunked\r\nDate: Sat, 22 Jun 2024 10:00:00 GMT\r\n\r\n64\r\nAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\r\n0\r\n\r\n'
+
 = HTTP decompression (gzip)
 
 conf.debug_dissector = True
@@ -248,11 +253,13 @@ for i in range(3, 10):
 = Test chunked with gzip
 
 conf.contribs["http"]["auto_compression"] = False
+conf.contribs["http"]["auto_chunk"] = False
 z = b'\x1f\x8b\x08\x00S\\-_\x02\xff\xb3\xc9(\xc9\xcd\xb1\xcb\xcd)\xb0\xd1\x07\xb3\x00\xe6\xedpt\x10\x00\x00\x00'
 a = IP(dst="1.1.1.1", src="2.2.2.2")/TCP(seq=1)/HTTP()/HTTPResponse(Content_Encoding="gzip", Transfer_Encoding="chunked")/(b"5\r\n" + z[:5] + b"\r\n")
 b = IP(dst="1.1.1.1", src="2.2.2.2")/TCP(seq=len(a[TCP].payload)+1)/HTTP()/(hex(len(z[5:])).encode()[2:] + b"\r\n" + z[5:] + b"\r\n0\r\n\r\n")
 xa, xb = IP(raw(a)), IP(raw(b))
 conf.contribs["http"]["auto_compression"] = True
+conf.contribs["http"]["auto_chunk"] = True
 
 c = sniff(offline=[xa, xb], session=TCPSession)[0]
 import gzip

test/scapy/layers/inet.uts

@@ -107,6 +107,10 @@ assert bytes_hex(bytes(buffer)) == b'0070696e6b696500706965'
 assert len(buffer) == 11
 assert buffer
 
+buffer = StringBuffer()
+buffer.append(b"")
+assert not buffer
+assert bytes(buffer) == b""
 
 ############
 ############