h3: remove HTTP/3 datagram API

An HTTP/3 datagram is a series of fields carried in a QUIC DATAGRAM frame
payload. In draft 00, draft 11 and RFC 9297 there is a single field,
which is a variable-length integer.

Previously, the h3 module provided support functions that were a thin shim over
quiche's transport-layer datagram methods, which helped with the variable-length
encoding of the first field in the HTTP/3 datagram. However, in the meantime we
spun out the Octets crate, which allows applications to easily handle varints
themselves. Furthermore, changes in specifications like CONNECT-UDP mean that
applications are probably required to do additional processing of varint fields
that might exist after the first field. At this stage, the h3 datagram API
provides minimal utility and in the worst case is confusing for applications
that would use it for different purposes.

This change removes quiche::h3::send_dgram() and quiche::h3::recv_dgram()
functions. Instead applications are now responsible for serialization /
deserialization of HTTP/3 datagrams, with the recommendation being that they use
the Octets crate. This is demonstrated by related changes to the apps.

Since the Octets crate does not provide a C API itself, helper function have
been added to quiche's C API to support varint handling.

apps/Cargo.toml

@@ -26,6 +26,7 @@ env_logger = "0.6"
 mio = { version = "0.8", features = ["net", "os-poll"] }
 url = "1"
 log = "0.4"
+octets = { version = "0.2", path = "../octets" }
 ring = "0.16"
 quiche = { path = "../quiche" }
 libc = "0.2"

apps/src/common.rs

@@ -313,6 +313,26 @@ pub fn priority_from_query_string(url: &url::Url) -> Option<Priority> {
     }
 }
 
+fn send_h3_dgram(
+    conn: &mut quiche::Connection, flow_id: u64, dgram_content: &[u8],
+) -> quiche::Result<()> {
+    info!(
+        "sending HTTP/3 DATAGRAM on flow_id={} with data {:?}",
+        flow_id, dgram_content
+    );
+
+    let len = octets::varint_len(flow_id) + dgram_content.len();
+    let mut d = vec![0; len];
+    let mut b = octets::OctetsMut::with_slice(&mut d);
+
+    b.put_varint(flow_id)
+        .map_err(|_| quiche::Error::BufferTooShort)?;
+    b.put_bytes(dgram_content)
+        .map_err(|_| quiche::Error::BufferTooShort)?;
+
+    conn.dgram_send(&d)
+}
+
 pub trait HttpConn {
     fn send_requests(
         &mut self, conn: &mut quiche::Connection, target_path: &Option<String>,
@@ -1177,17 +1197,8 @@ impl HttpConn for Http3Conn {
             let mut dgrams_done = 0;
 
             for _ in ds.dgrams_sent..ds.dgram_count {
-                info!(
-                    "sending HTTP/3 DATAGRAM on flow_id={} with data {:?}",
-                    ds.flow_id,
-                    ds.dgram_content.as_bytes()
-                );
-
-                match self.h3_conn.send_dgram(
-                    conn,
-                    0,
-                    ds.dgram_content.as_bytes(),
-                ) {
+                match send_h3_dgram(conn, ds.flow_id, ds.dgram_content.as_bytes())
+                {
                     Ok(v) => v,
 
                     Err(e) => {
@@ -1311,19 +1322,6 @@ impl HttpConn for Http3Conn {
                     break;
                 },
 
-                Ok((_flow_id, quiche::h3::Event::Datagram)) => {
-                    while let Ok((len, flow_id, flow_id_len)) =
-                        self.h3_conn.recv_dgram(conn, buf)
-                    {
-                        info!(
-                            "Received DATAGRAM flow_id={} len={} data={:?}",
-                            flow_id,
-                            len,
-                            buf[flow_id_len..len].to_vec()
-                        );
-                    }
-                },
-
                 Ok((
                     prioritized_element_id,
                     quiche::h3::Event::PriorityUpdate,
@@ -1354,6 +1352,19 @@ impl HttpConn for Http3Conn {
                 },
             }
         }
+
+        // Process datagram-related events.
+        while let Ok(len) = conn.dgram_recv(buf) {
+            let mut b = octets::Octets::with_slice(buf);
+            if let Ok(flow_id) = b.get_varint() {
+                info!(
+                    "Received DATAGRAM flow_id={} len={} data={:?}",
+                    flow_id,
+                    len,
+                    buf[b.off()..len].to_vec()
+                );
+            }
+        }
     }
 
     fn report_incomplete(&self, start: &std::time::Instant) -> bool {
@@ -1381,7 +1392,7 @@ impl HttpConn for Http3Conn {
         partial_responses: &mut HashMap<u64, PartialResponse>, root: &str,
         index: &str, buf: &mut [u8],
     ) -> quiche::h3::Result<()> {
-        // Process HTTP events.
+        // Process HTTP stream-related events.
         loop {
             match self.h3_conn.poll(conn) {
                 Ok((stream_id, quiche::h3::Event::Headers { list, .. })) => {
@@ -1506,19 +1517,6 @@ impl HttpConn for Http3Conn {
 
                 Ok((_stream_id, quiche::h3::Event::Reset { .. })) => (),
 
-                Ok((_, quiche::h3::Event::Datagram)) => {
-                    while let Ok((len, flow_id, flow_id_len)) =
-                        self.h3_conn.recv_dgram(conn, buf)
-                    {
-                        info!(
-                            "Received DATAGRAM flow_id={} len={} data={:?}",
-                            flow_id,
-                            len,
-                            buf[flow_id_len..len].to_vec()
-                        );
-                    }
-                },
-
                 Ok((
                     prioritized_element_id,
                     quiche::h3::Event::PriorityUpdate,
@@ -1552,21 +1550,25 @@ impl HttpConn for Http3Conn {
             }
         }
 
+        // Process datagram-related events.
+        while let Ok(len) = conn.dgram_recv(buf) {
+            let mut b = octets::Octets::with_slice(buf);
+            if let Ok(flow_id) = b.get_varint() {
+                info!(
+                    "Received DATAGRAM flow_id={} len={} data={:?}",
+                    flow_id,
+                    len,
+                    buf[b.off()..len].to_vec()
+                );
+            }
+        }
+
         if let Some(ds) = self.dgram_sender.as_mut() {
             let mut dgrams_done = 0;
 
             for _ in ds.dgrams_sent..ds.dgram_count {
-                info!(
-                    "sending HTTP/3 DATAGRAM on flow_id={} with data {:?}",
-                    ds.flow_id,
-                    ds.dgram_content.as_bytes()
-                );
-
-                match self.h3_conn.send_dgram(
-                    conn,
-                    0,
-                    ds.dgram_content.as_bytes(),
-                ) {
+                match send_h3_dgram(conn, ds.flow_id, ds.dgram_content.as_bytes())
+                {
                     Ok(v) => v,
 
                     Err(e) => {

nginx/nginx-1.16.patch

@@ -2264,7 +2264,7 @@ new file mode 100644
 index 000000000..d7c4fbf1d
 --- /dev/null
 +++ b/src/http/v3/ngx_http_v3.c
-@@ -0,0 +1,2273 @@
+@@ -0,0 +1,2270 @@
 +
 +/*
 + * Copyright (C) Cloudflare, Inc.
@@ -2748,9 +2748,6 @@ index 000000000..d7c4fbf1d
 +            case QUICHE_H3_EVENT_PRIORITY_UPDATE:
 +                break;
 +
-+            case QUICHE_H3_EVENT_DATAGRAM:
-+                break;
-+
 +            case QUICHE_H3_EVENT_GOAWAY:
 +                break;
 +        }

quiche/examples/http3-client.c

@@ -313,9 +313,6 @@ static void recv_cb(EV_P_ ev_io *w, int revents) {
                 case QUICHE_H3_EVENT_PRIORITY_UPDATE:
                     break;
 
-                case QUICHE_H3_EVENT_DATAGRAM:
-                    break;
-
                 case QUICHE_H3_EVENT_GOAWAY: {
                     fprintf(stderr, "got GOAWAY\n");
                     break;

quiche/examples/http3-client.rs

@@ -280,8 +280,6 @@ fn main() {
                         conn.close(true, 0x100, b"kthxbye").unwrap();
                     },
 
-                    Ok((_flow_id, quiche::h3::Event::Datagram)) => (),
-
                     Ok((_, quiche::h3::Event::PriorityUpdate)) => unreachable!(),
 
                     Ok((goaway_id, quiche::h3::Event::GoAway)) => {

quiche/examples/http3-server.c

@@ -459,9 +459,6 @@ static void recv_cb(EV_P_ ev_io *w, int revents) {
                     case QUICHE_H3_EVENT_PRIORITY_UPDATE:
                         break;
 
-                    case QUICHE_H3_EVENT_DATAGRAM:
-                        break;
-
                     case QUICHE_H3_EVENT_GOAWAY: {
                         fprintf(stderr, "got GOAWAY\n");
                         break;

quiche/examples/http3-server.rs

@@ -367,8 +367,6 @@ fn main() {
 
                         Ok((_stream_id, quiche::h3::Event::Reset { .. })) => (),
 
-                        Ok((_flow_id, quiche::h3::Event::Datagram)) => (),
-
                         Ok((
                             _prioritized_element_id,
                             quiche::h3::Event::PriorityUpdate,

quiche/include/quiche.h

@@ -662,6 +662,15 @@ ssize_t quiche_conn_send_ack_eliciting_on_path(quiche_conn *conn,
 // Frees the connection object.
 void quiche_conn_free(quiche_conn *conn);
 
+// Writes an unsigned variable-length integer in network byte-order into
+// the provided buffer.
+int quiche_put_varint(uint8_t *buf, size_t buf_len,
+                      uint64_t val);
+
+// Reads an unsigned variable-length integer in network byte-order from
+// the provided buffer and returns the wire length.
+ssize_t quiche_get_varint(const uint8_t *buf, size_t buf_len,
+                          uint64_t val);
 
 // HTTP/3 API
 //
@@ -830,7 +839,6 @@ enum quiche_h3_event_type {
     QUICHE_H3_EVENT_HEADERS,
     QUICHE_H3_EVENT_DATA,
     QUICHE_H3_EVENT_FINISHED,
-    QUICHE_H3_EVENT_DATAGRAM,
     QUICHE_H3_EVENT_GOAWAY,
     QUICHE_H3_EVENT_RESET,
     QUICHE_H3_EVENT_PRIORITY_UPDATE,
@@ -943,15 +951,6 @@ int quiche_h3_take_last_priority_update(quiche_h3_conn *conn,
 bool quiche_h3_dgram_enabled_by_peer(quiche_h3_conn *conn,
                                      quiche_conn *quic_conn);
 
-// Writes data to the DATAGRAM send queue.
-ssize_t quiche_h3_send_dgram(quiche_h3_conn *conn, quiche_conn *quic_conn,
-                            uint64_t flow_id, uint8_t *data, size_t data_len);
-
-// Reads data from the DATAGRAM receive queue.
-ssize_t quiche_h3_recv_dgram(quiche_h3_conn *conn, quiche_conn *quic_conn,
-                            uint64_t *flow_id, size_t *flow_id_len,
-                            uint8_t *out, size_t out_len);
-
 // Frees the HTTP/3 connection object.
 void quiche_h3_conn_free(quiche_h3_conn *conn);
 

quiche/src/ffi.rs

@@ -1328,6 +1328,42 @@ pub extern fn quiche_conn_send_quantum(conn: &Connection) -> size_t {
     conn.send_quantum() as size_t
 }
 
+#[no_mangle]
+pub extern fn quiche_put_varint(
+    buf: *mut u8, buf_len: size_t, val: u64,
+) -> c_int {
+    let buf = unsafe { slice::from_raw_parts_mut(buf, buf_len) };
+
+    let mut b = octets::OctetsMut::with_slice(buf);
+    match b.put_varint(val) {
+        Ok(_) => 0,
+
+        Err(e) => {
+            let err: Error = e.into();
+            err.to_c() as c_int
+        },
+    }
+}
+
+#[no_mangle]
+pub extern fn quiche_get_varint(
+    buf: *const u8, buf_len: size_t, val: *mut u64,
+) -> ssize_t {
+    let buf = unsafe { slice::from_raw_parts(buf, buf_len) };
+
+    let mut b = octets::Octets::with_slice(buf);
+    match b.get_varint() {
+        Ok(v) => unsafe { *val = v },
+
+        Err(e) => {
+            let err: Error = e.into();
+            return err.to_c();
+        },
+    };
+
+    b.off() as ssize_t
+}
+
 fn std_addr_from_c(addr: &sockaddr, addr_len: socklen_t) -> SocketAddr {
     match addr.sa_family as i32 {
         AF_INET => {

quiche/src/h3/ffi.rs

@@ -143,8 +143,6 @@ pub extern fn quiche_h3_event_type(ev: &h3::Event) -> u32 {
 
         h3::Event::Finished { .. } => 2,
 
-        h3::Event::Datagram { .. } => 3,
-
         h3::Event::GoAway { .. } => 4,
 
         h3::Event::Reset { .. } => 5,
@@ -365,46 +363,6 @@ pub extern fn quiche_h3_dgram_enabled_by_peer(
     conn.dgram_enabled_by_peer(quic_conn)
 }
 
-#[no_mangle]
-pub extern fn quiche_h3_send_dgram(
-    conn: &mut h3::Connection, quic_conn: &mut Connection, flow_id: u64,
-    data: *const u8, data_len: size_t,
-) -> c_int {
-    if data_len > <ssize_t>::max_value() as usize {
-        panic!("The provided buffer is too large");
-    }
-
-    let data = unsafe { slice::from_raw_parts(data, data_len) };
-
-    match conn.send_dgram(quic_conn, flow_id, data) {
-        Ok(_) => 0,
-
-        Err(e) => e.to_c() as c_int,
-    }
-}
-
-#[no_mangle]
-pub extern fn quiche_h3_recv_dgram(
-    conn: &mut h3::Connection, quic_conn: &mut Connection, flow_id: *mut u64,
-    flow_id_len: *mut usize, out: *mut u8, out_len: size_t,
-) -> ssize_t {
-    if out_len > <ssize_t>::max_value() as usize {
-        panic!("The provided buffer is too large");
-    }
-
-    let out = unsafe { slice::from_raw_parts_mut(out, out_len) };
-
-    match conn.recv_dgram(quic_conn, out) {
-        Ok((len, id, id_len)) => {
-            unsafe { *flow_id = id };
-            unsafe { *flow_id_len = id_len };
-            len as ssize_t
-        },
-
-        Err(e) => e.to_c(),
-    }
-}
-
 #[no_mangle]
 pub extern fn quiche_h3_conn_free(conn: *mut h3::Connection) {
     drop(unsafe { Box::from_raw(conn) });

quiche/src/h3/frame.rs

@@ -242,7 +242,7 @@ impl Frame {
 
                 if let Some(val) = h3_datagram {
                     b.put_varint(SETTINGS_H3_DATAGRAM_00)?;
-                    b.put_varint(*val as u64)?;
+                    b.put_varint(*val)?;
                     b.put_varint(SETTINGS_H3_DATAGRAM)?;
                     b.put_varint(*val)?;
                 }