diff --git a/deps/nghttp3/lib/nghttp3_conn.c b/deps/nghttp3/lib/nghttp3_conn.c index c3c1fb01a31b49..3d10f393bf0ff0 100644 --- a/deps/nghttp3/lib/nghttp3_conn.c +++ b/deps/nghttp3/lib/nghttp3_conn.c @@ -2351,12 +2351,13 @@ int nghttp3_conn_create_stream(nghttp3_conn *conn, nghttp3_stream **pstream, int nghttp3_conn_create_push_promise(nghttp3_conn *conn, nghttp3_push_promise **ppp, - int64_t push_id, nghttp3_tnode *parent) { + int64_t push_id, + nghttp3_tnode *assoc_tnode) { nghttp3_push_promise *pp; int rv; - rv = - nghttp3_push_promise_new(&pp, push_id, conn->next_seq, parent, conn->mem); + rv = nghttp3_push_promise_new(&pp, push_id, conn->next_seq, assoc_tnode, + conn->mem); if (rv != 0) { return rv; } @@ -3204,7 +3205,7 @@ void nghttp3_conn_settings_default(nghttp3_conn_settings *settings) { } int nghttp3_push_promise_new(nghttp3_push_promise **ppp, int64_t push_id, - uint64_t seq, nghttp3_tnode *parent, + uint64_t seq, nghttp3_tnode *assoc_tnode, const nghttp3_mem *mem) { nghttp3_push_promise *pp; nghttp3_node_id nid; @@ -3223,10 +3224,10 @@ int nghttp3_push_promise_new(nghttp3_push_promise **ppp, int64_t push_id, pp->http.status_code = -1; pp->http.content_length = -1; - if (parent) { - assert(parent->nid.type == NGHTTP3_NODE_ID_TYPE_STREAM); + if (assoc_tnode) { + assert(assoc_tnode->nid.type == NGHTTP3_NODE_ID_TYPE_STREAM); - pp->stream_id = parent->nid.id; + pp->stream_id = assoc_tnode->nid.id; pp->flags |= NGHTTP3_PUSH_PROMISE_FLAG_BOUND; } else { pp->stream_id = -1; diff --git a/deps/nghttp3/lib/nghttp3_conn.h b/deps/nghttp3/lib/nghttp3_conn.h index 81f7a6e22c2452..720b6102426acb 100644 --- a/deps/nghttp3/lib/nghttp3_conn.h +++ b/deps/nghttp3/lib/nghttp3_conn.h @@ -180,7 +180,8 @@ int nghttp3_conn_create_stream(nghttp3_conn *conn, nghttp3_stream **pstream, int nghttp3_conn_create_push_promise(nghttp3_conn *conn, nghttp3_push_promise **ppp, - int64_t push_id, nghttp3_tnode *parent); + int64_t push_id, + nghttp3_tnode *assoc_tnode); nghttp3_ssize nghttp3_conn_read_bidi(nghttp3_conn *conn, size_t *pnproc, nghttp3_stream *stream, const uint8_t *src, @@ -254,7 +255,7 @@ void nghttp3_conn_unschedule_stream(nghttp3_conn *conn, nghttp3_stream *stream); nghttp3_stream *nghttp3_conn_get_next_tx_stream(nghttp3_conn *conn); int nghttp3_push_promise_new(nghttp3_push_promise **ppp, int64_t push_id, - uint64_t seq, nghttp3_tnode *parent, + uint64_t seq, nghttp3_tnode *assoc_tnode, const nghttp3_mem *mem); void nghttp3_push_promise_del(nghttp3_push_promise *pp, const nghttp3_mem *mem); diff --git a/deps/nghttp3/lib/nghttp3_conv.h b/deps/nghttp3/lib/nghttp3_conv.h index 309332defedb2a..6fae9fc9684a44 100644 --- a/deps/nghttp3/lib/nghttp3_conv.h +++ b/deps/nghttp3/lib/nghttp3_conv.h @@ -38,6 +38,10 @@ # include #endif /* HAVE_NETINET_IN_H */ +#ifdef HAVE_BYTESWAP_H +# include +#endif /* HAVE_BYTESWAP_H */ + #ifdef HAVE_ENDIAN_H # include #endif /* HAVE_ENDIAN_H */ @@ -48,15 +52,25 @@ #include +#if defined HAVE_BSWAP_64 || HAVE_DECL_BSWAP_64 +# define nghttp3_bswap64 bswap_64 +#else /* !HAVE_BSWAP_64 */ +# define nghttp3_bswap64(N) \ + ((uint64_t)(ntohl((uint32_t)(N))) << 32 | ntohl((uint32_t)((N) >> 32))) +#endif /* !HAVE_BSWAP_64 */ + #if defined HAVE_BE64TOH || HAVE_DECL_BE64TOH # define nghttp3_ntohl64(N) be64toh(N) # define nghttp3_htonl64(N) htobe64(N) #else /* !HAVE_BE64TOH */ -# define nghttp3_bswap64(N) \ - ((uint64_t)(ntohl((uint32_t)(N))) << 32 | ntohl((uint32_t)((N) >> 32))) -# define nghttp3_ntohl64(N) nghttp3_bswap64(N) -# define nghttp3_htonl64(N) nghttp3_bswap64(N) -#endif /* !HAVE_BE64TOH */ +# if defined WORDS_BIGENDIAN +# define nghttp3_ntohl64(N) (N) +# define nghttp3_htonl64(N) (N) +# else /* !WORDS_BIGENDIAN */ +# define nghttp3_ntohl64(N) nghttp3_bswap64(N) +# define nghttp3_htonl64(N) nghttp3_bswap64(N) +# endif /* !WORDS_BIGENDIAN */ +#endif /* !HAVE_BE64TOH */ #if defined(WIN32) /* Windows requires ws2_32 library for ntonl family of functions. We diff --git a/deps/nghttp3/lib/nghttp3_ksl.c b/deps/nghttp3/lib/nghttp3_ksl.c index 958b03973c34f6..6a0f7c1de571cc 100644 --- a/deps/nghttp3/lib/nghttp3_ksl.c +++ b/deps/nghttp3/lib/nghttp3_ksl.c @@ -281,6 +281,13 @@ int nghttp3_ksl_insert(nghttp3_ksl *ksl, nghttp3_ksl_it *it, i = ksl_bsearch(ksl, blk, key, ksl->compar); if (blk->leaf) { + if (i < blk->n && + !ksl->compar(key, nghttp3_ksl_nth_node(ksl, blk, i)->key)) { + if (it) { + *it = nghttp3_ksl_end(ksl); + } + return NGHTTP3_ERR_INVALID_ARGUMENT; + } ksl_insert_node(ksl, blk, i, key, data); ++ksl->n; if (it) { @@ -453,8 +460,8 @@ static int key_equal(nghttp3_ksl_compar compar, const nghttp3_ksl_key *lhs, return !compar(lhs, rhs) && !compar(rhs, lhs); } -void nghttp3_ksl_remove(nghttp3_ksl *ksl, nghttp3_ksl_it *it, - const nghttp3_ksl_key *key) { +int nghttp3_ksl_remove(nghttp3_ksl *ksl, nghttp3_ksl_it *it, + const nghttp3_ksl_key *key) { nghttp3_ksl_blk *blk = ksl->head; nghttp3_ksl_node *node; size_t i; @@ -468,10 +475,20 @@ void nghttp3_ksl_remove(nghttp3_ksl *ksl, nghttp3_ksl_it *it, for (;;) { i = ksl_bsearch(ksl, blk, key, ksl->compar); - assert(i < blk->n); + if (i == blk->n) { + if (it) { + *it = nghttp3_ksl_end(ksl); + } + return NGHTTP3_ERR_INVALID_ARGUMENT; + } if (blk->leaf) { - assert(i < blk->n); + if (ksl->compar(key, nghttp3_ksl_nth_node(ksl, blk, i)->key)) { + if (it) { + *it = nghttp3_ksl_end(ksl); + } + return NGHTTP3_ERR_INVALID_ARGUMENT; + } ksl_remove_node(ksl, blk, i); --ksl->n; if (it) { @@ -481,7 +498,7 @@ void nghttp3_ksl_remove(nghttp3_ksl *ksl, nghttp3_ksl_it *it, nghttp3_ksl_it_init(it, ksl, blk, i); } } - return; + return 0; } node = nghttp3_ksl_nth_node(ksl, blk, i); diff --git a/deps/nghttp3/lib/nghttp3_ksl.h b/deps/nghttp3/lib/nghttp3_ksl.h index 88c636b0c1d9f8..aeb77eea575bcc 100644 --- a/deps/nghttp3/lib/nghttp3_ksl.h +++ b/deps/nghttp3/lib/nghttp3_ksl.h @@ -168,27 +168,33 @@ void nghttp3_ksl_free(nghttp3_ksl *ksl); * successful insertion, the iterator points to the inserted node is * stored in |*it|. * - * This function assumes that |key| does not exist in |ksl|. - * * This function returns 0 if it succeeds, or one of the following * negative error codes: * * NGHTTP3_ERR_NOMEM * Out of memory. + * NGHTTP3_ERR_INVALID_ARGUMENT + * |key| already exists. */ int nghttp3_ksl_insert(nghttp3_ksl *ksl, nghttp3_ksl_it *it, const nghttp3_ksl_key *key, void *data); /* - * nghttp3_ksl_remove removes the |key| from |ksl|. It assumes such - * the key is included in |ksl|. + * nghttp3_ksl_remove removes the |key| from |ksl|. * * This function assigns the iterator to |*it|, which points to the * node which is located at the right next of the removed node if |it| - * is not NULL. + * is not NULL. If |key| is not found, no deletion takes place and + * the return value of nghttp3_ksl_end(ksl) is assigned to |*it|. + * + * This function returns 0 if it succeeds, or one of the following + * negative error codes: + * + * NGHTTP3_ERR_INVALID_ARGUMENT + * |key| does not exist. */ -void nghttp3_ksl_remove(nghttp3_ksl *ksl, nghttp3_ksl_it *it, - const nghttp3_ksl_key *key); +int nghttp3_ksl_remove(nghttp3_ksl *ksl, nghttp3_ksl_it *it, + const nghttp3_ksl_key *key); /* * nghttp3_ksl_lower_bound returns the iterator which points to the diff --git a/deps/nghttp3/lib/nghttp3_map.c b/deps/nghttp3/lib/nghttp3_map.c index e383d2b0b8a3fc..e8eb811566dcc7 100644 --- a/deps/nghttp3/lib/nghttp3_map.c +++ b/deps/nghttp3/lib/nghttp3_map.c @@ -27,6 +27,7 @@ #include "nghttp3_map.h" #include +#include #include "nghttp3_conv.h" @@ -36,7 +37,7 @@ int nghttp3_map_init(nghttp3_map *map, const nghttp3_mem *mem) { map->mem = mem; map->tablelen = INITIAL_TABLE_LENGTH; map->table = - nghttp3_mem_calloc(mem, map->tablelen, sizeof(nghttp3_map_entry *)); + nghttp3_mem_calloc(mem, map->tablelen, sizeof(nghttp3_map_bucket)); if (map->table == NULL) { return NGHTTP3_ERR_NOMEM; } @@ -47,38 +48,82 @@ int nghttp3_map_init(nghttp3_map *map, const nghttp3_mem *mem) { } void nghttp3_map_free(nghttp3_map *map) { + size_t i; + nghttp3_map_bucket *bkt; + + if (!map) { + return; + } + + for (i = 0; i < map->tablelen; ++i) { + bkt = &map->table[i]; + if (bkt->ksl) { + nghttp3_ksl_free(bkt->ksl); + nghttp3_mem_free(map->mem, bkt->ksl); + } + } + nghttp3_mem_free(map->mem, map->table); } -void nghttp3_map_each_free(const nghttp3_map *map, +void nghttp3_map_each_free(nghttp3_map *map, int (*func)(nghttp3_map_entry *entry, void *ptr), void *ptr) { uint32_t i; + nghttp3_map_bucket *bkt; + nghttp3_ksl_it it; + for (i = 0; i < map->tablelen; ++i) { - nghttp3_map_entry *entry; - for (entry = map->table[i]; entry;) { - nghttp3_map_entry *next = entry->next; - func(entry, ptr); - entry = next; + bkt = &map->table[i]; + + if (bkt->ptr) { + func(bkt->ptr, ptr); + bkt->ptr = NULL; + assert(bkt->ksl == NULL || nghttp3_ksl_len(bkt->ksl) == 0); + continue; + } + + if (bkt->ksl) { + for (it = nghttp3_ksl_begin(bkt->ksl); !nghttp3_ksl_it_end(&it); + nghttp3_ksl_it_next(&it)) { + func(nghttp3_ksl_it_get(&it), ptr); + } + + nghttp3_ksl_free(bkt->ksl); + nghttp3_mem_free(map->mem, bkt->ksl); + bkt->ksl = NULL; } - map->table[i] = NULL; } } -int nghttp3_map_each(const nghttp3_map *map, +int nghttp3_map_each(nghttp3_map *map, int (*func)(nghttp3_map_entry *entry, void *ptr), void *ptr) { int rv; uint32_t i; + nghttp3_map_bucket *bkt; + nghttp3_ksl_it it; + for (i = 0; i < map->tablelen; ++i) { - nghttp3_map_entry *entry, *next; - for (entry = map->table[i]; entry;) { - next = entry->next; - rv = func(entry, ptr); + bkt = &map->table[i]; + + if (bkt->ptr) { + rv = func(bkt->ptr, ptr); if (rv != 0) { return rv; } - entry = next; + assert(bkt->ksl == NULL || nghttp3_ksl_len(bkt->ksl) == 0); + continue; + } + + if (bkt->ksl) { + for (it = nghttp3_ksl_begin(bkt->ksl); !nghttp3_ksl_it_end(&it); + nghttp3_ksl_it_next(&it)) { + rv = func(nghttp3_ksl_it_get(&it), ptr); + if (rv != 0) { + return rv; + } + } } } return 0; @@ -98,71 +143,125 @@ static uint32_t hash(key_type key, uint32_t mod) { p = (uint8_t *)&key; end = p + sizeof(key_type); - for (; p != end; ++p) { - h ^= *p; - h *= 0x01000193u; + for (; p != end;) { + h ^= *p++; + h += (h << 1) + (h << 4) + (h << 7) + (h << 8) + (h << 24); } return h & (mod - 1); } -static int insert(nghttp3_map_entry **table, uint32_t tablelen, - nghttp3_map_entry *entry) { +static int less(const nghttp3_ksl_key *lhs, const nghttp3_ksl_key *rhs) { + return *(key_type *)lhs < *(key_type *)rhs; +} + +static int map_insert(nghttp3_map *map, nghttp3_map_bucket *table, + uint32_t tablelen, nghttp3_map_entry *entry) { uint32_t h = hash(entry->key, tablelen); - if (table[h] == NULL) { - table[h] = entry; - } else { - nghttp3_map_entry *p; - /* We won't allow duplicated key, so check it out. */ - for (p = table[h]; p; p = p->next) { - if (p->key == entry->key) { - return NGHTTP3_ERR_INVALID_ARGUMENT; - } + nghttp3_map_bucket *bkt = &table[h]; + const nghttp3_mem *mem = map->mem; + int rv; + + if (bkt->ptr == NULL && + (bkt->ksl == NULL || nghttp3_ksl_len(bkt->ksl) == 0)) { + bkt->ptr = entry; + return 0; + } + + if (!bkt->ksl) { + bkt->ksl = nghttp3_mem_malloc(mem, sizeof(*bkt->ksl)); + if (bkt->ksl == NULL) { + return NGHTTP3_ERR_NOMEM; } - entry->next = table[h]; - table[h] = entry; + nghttp3_ksl_init(bkt->ksl, less, sizeof(key_type), mem); } - return 0; + + if (bkt->ptr) { + rv = nghttp3_ksl_insert(bkt->ksl, NULL, &bkt->ptr->key, bkt->ptr); + if (rv != 0) { + return rv; + } + + bkt->ptr = NULL; + } + + return nghttp3_ksl_insert(bkt->ksl, NULL, &entry->key, entry); } /* new_tablelen must be power of 2 */ -static int resize(nghttp3_map *map, uint32_t new_tablelen) { +static int map_resize(nghttp3_map *map, uint32_t new_tablelen) { uint32_t i; - nghttp3_map_entry **new_table; + nghttp3_map_bucket *new_table; + nghttp3_map_bucket *bkt; + nghttp3_ksl_it it; + int rv; new_table = - nghttp3_mem_calloc(map->mem, new_tablelen, sizeof(nghttp3_map_entry *)); + nghttp3_mem_calloc(map->mem, new_tablelen, sizeof(nghttp3_map_bucket)); if (new_table == NULL) { return NGHTTP3_ERR_NOMEM; } for (i = 0; i < map->tablelen; ++i) { - nghttp3_map_entry *entry; - for (entry = map->table[i]; entry;) { - nghttp3_map_entry *next = entry->next; - entry->next = NULL; - /* This function must succeed */ - insert(new_table, new_tablelen, entry); - entry = next; + bkt = &map->table[i]; + + if (bkt->ptr) { + rv = map_insert(map, new_table, new_tablelen, bkt->ptr); + if (rv != 0) { + goto fail; + } + assert(bkt->ksl == NULL || nghttp3_ksl_len(bkt->ksl) == 0); + continue; + } + + if (bkt->ksl) { + for (it = nghttp3_ksl_begin(bkt->ksl); !nghttp3_ksl_it_end(&it); + nghttp3_ksl_it_next(&it)) { + rv = map_insert(map, new_table, new_tablelen, nghttp3_ksl_it_get(&it)); + if (rv != 0) { + goto fail; + } + } + } + } + + for (i = 0; i < map->tablelen; ++i) { + bkt = &map->table[i]; + if (bkt->ksl) { + nghttp3_ksl_free(bkt->ksl); + nghttp3_mem_free(map->mem, bkt->ksl); } } + nghttp3_mem_free(map->mem, map->table); map->tablelen = new_tablelen; map->table = new_table; return 0; + +fail: + for (i = 0; i < new_tablelen; ++i) { + bkt = &new_table[i]; + if (bkt->ksl) { + nghttp3_ksl_free(bkt->ksl); + nghttp3_mem_free(map->mem, bkt->ksl); + } + } + + return rv; } int nghttp3_map_insert(nghttp3_map *map, nghttp3_map_entry *new_entry) { int rv; + /* Load factor is 0.75 */ if ((map->size + 1) * 4 > map->tablelen * 3) { - rv = resize(map, map->tablelen * 2); + rv = map_resize(map, map->tablelen * 2); if (rv != 0) { return rv; } } - rv = insert(map->table, map->tablelen, new_entry); + rv = map_insert(map, map->table, map->tablelen, new_entry); if (rv != 0) { return rv; } @@ -170,44 +269,69 @@ int nghttp3_map_insert(nghttp3_map *map, nghttp3_map_entry *new_entry) { return 0; } -nghttp3_map_entry *nghttp3_map_find(const nghttp3_map *map, key_type key) { - uint32_t h; - nghttp3_map_entry *entry; - h = hash(key, map->tablelen); - for (entry = map->table[h]; entry; entry = entry->next) { - if (entry->key == key) { - return entry; +nghttp3_map_entry *nghttp3_map_find(nghttp3_map *map, key_type key) { + nghttp3_map_bucket *bkt = &map->table[hash(key, map->tablelen)]; + nghttp3_ksl_it it; + + if (bkt->ptr) { + if (bkt->ptr->key == key) { + return bkt->ptr; } + return NULL; } + + if (bkt->ksl) { + it = nghttp3_ksl_lower_bound(bkt->ksl, &key); + if (nghttp3_ksl_it_end(&it) || + *(key_type *)nghttp3_ksl_it_key(&it) != key) { + return NULL; + } + return nghttp3_ksl_it_get(&it); + } + return NULL; } int nghttp3_map_remove(nghttp3_map *map, key_type key) { - uint32_t h; - nghttp3_map_entry **dst; - - h = hash(key, map->tablelen); + nghttp3_map_bucket *bkt = &map->table[hash(key, map->tablelen)]; + int rv; - for (dst = &map->table[h]; *dst; dst = &(*dst)->next) { - if ((*dst)->key != key) { - continue; + if (bkt->ptr) { + if (bkt->ptr->key == key) { + bkt->ptr = NULL; + --map->size; + return 0; } + return NGHTTP3_ERR_INVALID_ARGUMENT; + } - *dst = (*dst)->next; + if (bkt->ksl) { + rv = nghttp3_ksl_remove(bkt->ksl, NULL, &key); + if (rv != 0) { + return rv; + } --map->size; return 0; } + return NGHTTP3_ERR_INVALID_ARGUMENT; } void nghttp3_map_clear(nghttp3_map *map) { uint32_t i; + nghttp3_map_bucket *bkt; for (i = 0; i < map->tablelen; ++i) { - map->table[i] = NULL; + bkt = &map->table[i]; + bkt->ptr = NULL; + if (bkt->ksl) { + nghttp3_ksl_free(bkt->ksl); + nghttp3_mem_free(map->mem, bkt->ksl); + bkt->ksl = NULL; + } } map->size = 0; } -size_t nghttp3_map_size(const nghttp3_map *map) { return map->size; } +size_t nghttp3_map_size(nghttp3_map *map) { return map->size; } diff --git a/deps/nghttp3/lib/nghttp3_map.h b/deps/nghttp3/lib/nghttp3_map.h index 0b7c0d28c37c7f..8a00711970907c 100644 --- a/deps/nghttp3/lib/nghttp3_map.h +++ b/deps/nghttp3/lib/nghttp3_map.h @@ -34,6 +34,7 @@ #include #include "nghttp3_mem.h" +#include "nghttp3_ksl.h" /* Implementation of unordered map */ @@ -44,8 +45,13 @@ typedef struct nghttp3_map_entry { key_type key; } nghttp3_map_entry; +typedef struct nghttp3_map_bucket { + nghttp3_map_entry *ptr; + nghttp3_ksl *ksl; +} nghttp3_map_bucket; + typedef struct { - nghttp3_map_entry **table; + nghttp3_map_bucket *table; const nghttp3_mem *mem; size_t size; uint32_t tablelen; @@ -75,7 +81,7 @@ void nghttp3_map_free(nghttp3_map *map); * given the |entry| object. The |ptr| will be passed to the |func| as * send argument. The return value of the |func| will be ignored. */ -void nghttp3_map_each_free(const nghttp3_map *map, +void nghttp3_map_each_free(nghttp3_map *map, int (*func)(nghttp3_map_entry *entry, void *ptr), void *ptr); @@ -102,7 +108,7 @@ int nghttp3_map_insert(nghttp3_map *map, nghttp3_map_entry *entry); * Returns the entry associated by the key |key|. If there is no such * entry, this function returns NULL. */ -nghttp3_map_entry *nghttp3_map_find(const nghttp3_map *map, key_type key); +nghttp3_map_entry *nghttp3_map_find(nghttp3_map *map, key_type key); /* * Removes the entry associated by the key |key| from the |map|. The @@ -124,7 +130,7 @@ void nghttp3_map_clear(nghttp3_map *map); /* * Returns the number of items stored in the map |map|. */ -size_t nghttp3_map_size(const nghttp3_map *map); +size_t nghttp3_map_size(nghttp3_map *map); /* * Applies the function |func| to each entry in the |map| with the @@ -140,7 +146,7 @@ size_t nghttp3_map_size(const nghttp3_map *map); * Don't use this function to free each entry. Use * nghttp3_map_each_free() instead. */ -int nghttp3_map_each(const nghttp3_map *map, +int nghttp3_map_each(nghttp3_map *map, int (*func)(nghttp3_map_entry *entry, void *ptr), void *ptr); diff --git a/deps/nghttp3/lib/nghttp3_qpack.c b/deps/nghttp3/lib/nghttp3_qpack.c index 370c3eb442bd8e..d74e29f6fec916 100644 --- a/deps/nghttp3/lib/nghttp3_qpack.c +++ b/deps/nghttp3/lib/nghttp3_qpack.c @@ -1043,8 +1043,10 @@ void nghttp3_qpack_encoder_shrink_dtable(nghttp3_qpack_encoder *encoder) { /* * qpack_encoder_add_stream_ref adds another dynamic table reference - * to a stream denoted by |stream_id|. |max_cnt| and |min_cnt| is the - * maximum and minimum insert count it references respectively. + * to a stream denoted by |stream_id|. |stream| must be NULL if no + * stream object is not found for the given stream ID. |max_cnt| and + * |min_cnt| is the maximum and minimum insert count it references + * respectively. * * This function returns 0 if it succeeds, or one of the following * negative error codes: @@ -1053,10 +1055,9 @@ void nghttp3_qpack_encoder_shrink_dtable(nghttp3_qpack_encoder *encoder) { * Out of memory. */ static int qpack_encoder_add_stream_ref(nghttp3_qpack_encoder *encoder, - int64_t stream_id, uint64_t max_cnt, - uint64_t min_cnt) { - nghttp3_qpack_stream *stream = - nghttp3_qpack_encoder_find_stream(encoder, stream_id); + int64_t stream_id, + nghttp3_qpack_stream *stream, + uint64_t max_cnt, uint64_t min_cnt) { nghttp3_qpack_header_block_ref *ref; const nghttp3_mem *mem = encoder->ctx.mem; uint64_t prev_max_cnt = 0; @@ -1211,7 +1212,8 @@ int nghttp3_qpack_encoder_encode(nghttp3_qpack_encoder *encoder, return 0; } - rv = qpack_encoder_add_stream_ref(encoder, stream_id, max_cnt, min_cnt); + rv = qpack_encoder_add_stream_ref(encoder, stream_id, stream, max_cnt, + min_cnt); if (rv != 0) { goto fail; } @@ -1308,19 +1310,6 @@ int nghttp3_qpack_encoder_stream_is_blocked(nghttp3_qpack_encoder *encoder, return stream && encoder->krcnt < nghttp3_qpack_stream_get_max_cnt(stream); } -static uint32_t qpack_hash_name(const nghttp3_nv *nv) { - /* 32 bit FNV-1a: http://isthe.com/chongo/tech/comp/fnv/ */ - uint32_t h = 2166136261u; - size_t i; - - for (i = 0; i < nv->namelen; ++i) { - h ^= nv->name[i]; - h += (h << 1) + (h << 4) + (h << 7) + (h << 8) + (h << 24); - } - - return h; -} - /* * qpack_encoder_decide_indexing_mode determines and returns indexing * mode for header field |nv|. |token| is a token of header field @@ -1341,6 +1330,7 @@ qpack_encoder_decide_indexing_mode(nghttp3_qpack_encoder *encoder, return NGHTTP3_QPACK_INDEXING_MODE_NEVER; } break; + case -1: case NGHTTP3_QPACK_TOKEN__PATH: case NGHTTP3_QPACK_TOKEN_AGE: case NGHTTP3_QPACK_TOKEN_CONTENT_LENGTH: @@ -1350,6 +1340,15 @@ qpack_encoder_decide_indexing_mode(nghttp3_qpack_encoder *encoder, case NGHTTP3_QPACK_TOKEN_LOCATION: case NGHTTP3_QPACK_TOKEN_SET_COOKIE: return NGHTTP3_QPACK_INDEXING_MODE_LITERAL; + case NGHTTP3_QPACK_TOKEN_HOST: + case NGHTTP3_QPACK_TOKEN_TE: + case NGHTTP3_QPACK_TOKEN__PROTOCOL: + case NGHTTP3_QPACK_TOKEN_PRIORITY: + break; + default: + if (token >= 1000) { + return NGHTTP3_QPACK_INDEXING_MODE_LITERAL; + } } if (table_space(nv->namelen, nv->valuelen) > @@ -1461,7 +1460,20 @@ int nghttp3_qpack_encoder_encode_nv(nghttp3_qpack_encoder *encoder, if (static_entry) { hash = token_stable[token].hash; } else { - hash = qpack_hash_name(nv); + switch (token) { + case NGHTTP3_QPACK_TOKEN_HOST: + hash = 2952701295u; + break; + case NGHTTP3_QPACK_TOKEN_TE: + hash = 1011170994u; + break; + case NGHTTP3_QPACK_TOKEN__PROTOCOL: + hash = 1128642621u; + break; + case NGHTTP3_QPACK_TOKEN_PRIORITY: + hash = 2498028297u; + break; + } } indexing_mode = qpack_encoder_decide_indexing_mode(encoder, nv, token); @@ -1474,7 +1486,8 @@ int nghttp3_qpack_encoder_encode_nv(nghttp3_qpack_encoder *encoder, } } - if (nghttp3_map_size(&encoder->streams) < NGHTTP3_QPACK_MAX_QPACK_STREAMS) { + if (hash && + nghttp3_map_size(&encoder->streams) < NGHTTP3_QPACK_MAX_QPACK_STREAMS) { dres = nghttp3_qpack_encoder_lookup_dtable(encoder, nv, token, hash, indexing_mode, encoder->krcnt, allow_blocking); @@ -2949,6 +2962,7 @@ nghttp3_ssize nghttp3_qpack_decoder_read_encoder(nghttp3_qpack_decoder *decoder, qpack_read_state_terminate_name(&decoder->rstate); decoder->state = NGHTTP3_QPACK_ES_STATE_CHECK_VALUE_HUFFMAN; + decoder->rstate.prefix = 7; break; case NGHTTP3_QPACK_ES_STATE_CHECK_VALUE_HUFFMAN: qpack_read_state_check_huffman(&decoder->rstate, *p); diff --git a/deps/nghttp3/lib/nghttp3_qpack.h b/deps/nghttp3/lib/nghttp3_qpack.h index 1a3097cae8dbf4..ddfb2b66385f8c 100644 --- a/deps/nghttp3/lib/nghttp3_qpack.h +++ b/deps/nghttp3/lib/nghttp3_qpack.h @@ -195,7 +195,7 @@ void nghttp3_qpack_read_state_free(nghttp3_qpack_read_state *rstate); void nghttp3_qpack_read_state_reset(nghttp3_qpack_read_state *rstate); -#define NGHTTP3_QPACK_MAP_SIZE 128 +#define NGHTTP3_QPACK_MAP_SIZE 64 typedef struct { nghttp3_qpack_entry *table[NGHTTP3_QPACK_MAP_SIZE]; diff --git a/deps/nghttp3/lib/nghttp3_ringbuf.c b/deps/nghttp3/lib/nghttp3_ringbuf.c index 6eea9830da17e0..9ea91c81c8a1b9 100644 --- a/deps/nghttp3/lib/nghttp3_ringbuf.c +++ b/deps/nghttp3/lib/nghttp3_ringbuf.c @@ -33,6 +33,16 @@ #include "nghttp3_macro.h" +#if defined(_MSC_VER) && defined(_M_ARM64) +unsigned int __popcnt(unsigned int x) { + unsigned int c = 0; + for (; x; ++c) { + x &= x - 1; + } + return c; +} +#endif + int nghttp3_ringbuf_init(nghttp3_ringbuf *rb, size_t nmemb, size_t size, const nghttp3_mem *mem) { if (nmemb) { @@ -107,8 +117,6 @@ void *nghttp3_ringbuf_get(nghttp3_ringbuf *rb, size_t offset) { return &rb->buf[offset * rb->size]; } -size_t nghttp3_ringbuf_len(nghttp3_ringbuf *rb) { return rb->len; } - int nghttp3_ringbuf_full(nghttp3_ringbuf *rb) { return rb->len == rb->nmemb; } int nghttp3_ringbuf_reserve(nghttp3_ringbuf *rb, size_t nmemb) { diff --git a/deps/nghttp3/lib/nghttp3_ringbuf.h b/deps/nghttp3/lib/nghttp3_ringbuf.h index d11bb2b0318aae..51194bd63569d5 100644 --- a/deps/nghttp3/lib/nghttp3_ringbuf.h +++ b/deps/nghttp3/lib/nghttp3_ringbuf.h @@ -103,7 +103,7 @@ void nghttp3_ringbuf_resize(nghttp3_ringbuf *rb, size_t len); void *nghttp3_ringbuf_get(nghttp3_ringbuf *rb, size_t offset); /* nghttp3_ringbuf_len returns the number of elements stored. */ -size_t nghttp3_ringbuf_len(nghttp3_ringbuf *rb); +#define nghttp3_ringbuf_len(RB) ((RB)->len) /* nghttp3_ringbuf_full returns nonzero if |rb| is full. */ int nghttp3_ringbuf_full(nghttp3_ringbuf *rb); diff --git a/deps/ngtcp2/crypto/includes/ngtcp2/ngtcp2_crypto.h b/deps/ngtcp2/crypto/includes/ngtcp2/ngtcp2_crypto.h index 3197c5dcb86729..4b6885a08e4dbe 100644 --- a/deps/ngtcp2/crypto/includes/ngtcp2/ngtcp2_crypto.h +++ b/deps/ngtcp2/crypto/includes/ngtcp2/ngtcp2_crypto.h @@ -210,8 +210,9 @@ NGTCP2_EXTERN int ngtcp2_crypto_derive_packet_protection_key( */ NGTCP2_EXTERN int ngtcp2_crypto_encrypt(uint8_t *dest, const ngtcp2_crypto_aead *aead, + const ngtcp2_crypto_aead_ctx *aead_ctx, const uint8_t *plaintext, - size_t plaintextlen, const uint8_t *key, + size_t plaintextlen, const uint8_t *nonce, size_t noncelen, const uint8_t *ad, size_t adlen); @@ -227,9 +228,10 @@ NGTCP2_EXTERN int ngtcp2_crypto_encrypt(uint8_t *dest, */ NGTCP2_EXTERN int ngtcp2_crypto_encrypt_cb(uint8_t *dest, const ngtcp2_crypto_aead *aead, + const ngtcp2_crypto_aead_ctx *aead_ctx, const uint8_t *plaintext, size_t plaintextlen, - const uint8_t *key, const uint8_t *nonce, - size_t noncelen, const uint8_t *ad, size_t adlen); + const uint8_t *nonce, size_t noncelen, + const uint8_t *ad, size_t adlen); /** * @function @@ -243,11 +245,13 @@ ngtcp2_crypto_encrypt_cb(uint8_t *dest, const ngtcp2_crypto_aead *aead, * * This function returns 0 if it succeeds, or -1. */ -NGTCP2_EXTERN int -ngtcp2_crypto_decrypt(uint8_t *dest, const ngtcp2_crypto_aead *aead, - const uint8_t *ciphertext, size_t ciphertextlen, - const uint8_t *key, const uint8_t *nonce, size_t noncelen, - const uint8_t *ad, size_t adlen); +NGTCP2_EXTERN int ngtcp2_crypto_decrypt(uint8_t *dest, + const ngtcp2_crypto_aead *aead, + const ngtcp2_crypto_aead_ctx *aead_ctx, + const uint8_t *ciphertext, + size_t ciphertextlen, + const uint8_t *nonce, size_t noncelen, + const uint8_t *ad, size_t adlen); /** * @function @@ -261,9 +265,10 @@ ngtcp2_crypto_decrypt(uint8_t *dest, const ngtcp2_crypto_aead *aead, */ NGTCP2_EXTERN int ngtcp2_crypto_decrypt_cb(uint8_t *dest, const ngtcp2_crypto_aead *aead, + const ngtcp2_crypto_aead_ctx *aead_ctx, const uint8_t *ciphertext, size_t ciphertextlen, - const uint8_t *key, const uint8_t *nonce, - size_t noncelen, const uint8_t *ad, size_t adlen); + const uint8_t *nonce, size_t noncelen, + const uint8_t *ad, size_t adlen); /** * @function @@ -277,7 +282,7 @@ ngtcp2_crypto_decrypt_cb(uint8_t *dest, const ngtcp2_crypto_aead *aead, */ NGTCP2_EXTERN int ngtcp2_crypto_hp_mask(uint8_t *dest, const ngtcp2_crypto_cipher *hp, - const uint8_t *key, + const ngtcp2_crypto_cipher_ctx *hp_ctx, const uint8_t *sample); /** @@ -290,10 +295,10 @@ NGTCP2_EXTERN int ngtcp2_crypto_hp_mask(uint8_t *dest, * This function returns 0 if it succeeds, or * :enum:`NGTCP2_ERR_CALLBACK_FAILURE`. */ -NGTCP2_EXTERN int ngtcp2_crypto_hp_mask_cb(uint8_t *dest, - const ngtcp2_crypto_cipher *hp, - const uint8_t *key, - const uint8_t *sample); +NGTCP2_EXTERN int +ngtcp2_crypto_hp_mask_cb(uint8_t *dest, const ngtcp2_crypto_cipher *hp, + const ngtcp2_crypto_cipher_ctx *hp_ctx, + const uint8_t *sample); /** * @function @@ -381,10 +386,12 @@ NGTCP2_EXTERN int ngtcp2_crypto_derive_and_install_tx_key( * The derived packet protection key for decryption is written to the * buffer pointed by |rx_key|. The derived packet protection IV for * decryption is written to the buffer pointed by |rx_iv|. + * |rx_aead_ctx| must be constructed with |rx_key|. * * The derived packet protection key for encryption is written to the * buffer pointed by |tx_key|. The derived packet protection IV for * encryption is written to the buffer pointed by |tx_iv|. + * |tx_aead_ctx| must be constructed with |rx_key|. * * |current_rx_secret| and |current_tx_secret| are the current traffic * secrets for decryption and encryption. |secretlen| specifies the @@ -397,12 +404,12 @@ NGTCP2_EXTERN int ngtcp2_crypto_derive_and_install_tx_key( * * This function returns 0 if it succeeds, or -1. */ -NGTCP2_EXTERN int -ngtcp2_crypto_update_key(ngtcp2_conn *conn, uint8_t *rx_secret, - uint8_t *tx_secret, uint8_t *rx_key, uint8_t *rx_iv, - uint8_t *tx_key, uint8_t *tx_iv, - const uint8_t *current_rx_secret, - const uint8_t *current_tx_secret, size_t secretlen); +NGTCP2_EXTERN int ngtcp2_crypto_update_key( + ngtcp2_conn *conn, uint8_t *rx_secret, uint8_t *tx_secret, + ngtcp2_crypto_aead_ctx *rx_aead_ctx, uint8_t *rx_key, uint8_t *rx_iv, + ngtcp2_crypto_aead_ctx *tx_aead_ctx, uint8_t *tx_key, uint8_t *tx_iv, + const uint8_t *current_rx_secret, const uint8_t *current_tx_secret, + size_t secretlen); /** * @function @@ -415,8 +422,9 @@ ngtcp2_crypto_update_key(ngtcp2_conn *conn, uint8_t *rx_secret, * :enum:`NGTCP2_ERR_CALLBACK_FAILURE`. */ NGTCP2_EXTERN int ngtcp2_crypto_update_key_cb( - ngtcp2_conn *conn, uint8_t *rx_secret, uint8_t *tx_secret, uint8_t *rx_key, - uint8_t *rx_iv, uint8_t *tx_key, uint8_t *tx_iv, + ngtcp2_conn *conn, uint8_t *rx_secret, uint8_t *tx_secret, + ngtcp2_crypto_aead_ctx *rx_aead_ctx, uint8_t *rx_iv, + ngtcp2_crypto_aead_ctx *tx_aead_ctx, uint8_t *tx_iv, const uint8_t *current_rx_secret, const uint8_t *current_tx_secret, size_t secretlen, void *user_data); @@ -543,6 +551,69 @@ ngtcp2_crypto_write_retry(uint8_t *dest, size_t destlen, const ngtcp2_cid *dcid, const ngtcp2_cid *scid, const ngtcp2_cid *odcid, const uint8_t *token, size_t tokenlen); +/** + * @function + * + * `ngtcp2_crypto_aead_ctx_encrypt_init` initializes |aead_ctx| with + * new AEAD cipher context object for encryption which is constructed + * to use |key| as encryption key. |aead| specifies AEAD cipher to + * use. |noncelen| is the length of nonce. + * + * This function returns 0 if it succeeds, or -1. + */ +NGTCP2_EXTERN int +ngtcp2_crypto_aead_ctx_encrypt_init(ngtcp2_crypto_aead_ctx *aead_ctx, + const ngtcp2_crypto_aead *aead, + const uint8_t *key, size_t noncelen); + +/** + * @function + * + * `ngtcp2_crypto_aead_ctx_decrypt_init` initializes |aead_ctx| with + * new AEAD cipher context object for decryption which is constructed + * to use |key| as encryption key. |aead| specifies AEAD cipher to + * use. |noncelen| is the length of nonce. + * + * This function returns 0 if it succeeds, or -1. + */ +NGTCP2_EXTERN int +ngtcp2_crypto_aead_ctx_decrypt_init(ngtcp2_crypto_aead_ctx *aead_ctx, + const ngtcp2_crypto_aead *aead, + const uint8_t *key, size_t noncelen); + +/** + * @function + * + * `ngtcp2_crypto_aead_ctx_free` frees up resources used by + * |aead_ctx|. This function does not free the memory pointed by + * |aead_ctx| itself. + */ +NGTCP2_EXTERN void +ngtcp2_crypto_aead_ctx_free(ngtcp2_crypto_aead_ctx *aead_ctx); + +/** + * @function + * + * `ngtcp2_crypto_delete_crypto_aead_ctx_cb` deletes the given |aead_ctx|. + * + * This function can be directly passed to delete_crypto_aead_ctx + * field in ngtcp2_callbacks. + */ +NGTCP2_EXTERN void ngtcp2_crypto_delete_crypto_aead_ctx_cb( + ngtcp2_conn *conn, ngtcp2_crypto_aead_ctx *aead_ctx, void *user_data); + +/** + * @function + * + * `ngtcp2_crypto_delete_crypto_cipher_ctx_cb` deletes the given + * |cipher_ctx|. + * + * This function can be directly passed to delete_crypto_cipher_ctx + * field in ngtcp2_callbacks. + */ +NGTCP2_EXTERN void ngtcp2_crypto_delete_crypto_cipher_ctx_cb( + ngtcp2_conn *conn, ngtcp2_crypto_cipher_ctx *cipher_ctx, void *user_data); + #ifdef __cplusplus } #endif diff --git a/deps/ngtcp2/crypto/openssl/openssl.c b/deps/ngtcp2/crypto/openssl/openssl.c index 60e7250eae9dbe..d1937441efb7ef 100644 --- a/deps/ngtcp2/crypto/openssl/openssl.c +++ b/deps/ngtcp2/crypto/openssl/openssl.c @@ -173,6 +173,92 @@ size_t ngtcp2_crypto_aead_taglen(const ngtcp2_crypto_aead *aead) { return crypto_aead_taglen(aead->native_handle); } +int ngtcp2_crypto_aead_ctx_encrypt_init(ngtcp2_crypto_aead_ctx *aead_ctx, + const ngtcp2_crypto_aead *aead, + const uint8_t *key, size_t noncelen) { + const EVP_CIPHER *cipher = aead->native_handle; + EVP_CIPHER_CTX *actx; + + actx = EVP_CIPHER_CTX_new(); + if (actx == NULL) { + return -1; + } + + if (!EVP_EncryptInit_ex(actx, cipher, NULL, NULL, NULL) || + !EVP_CIPHER_CTX_ctrl(actx, EVP_CTRL_AEAD_SET_IVLEN, (int)noncelen, + NULL) || + (cipher == EVP_aes_128_ccm() && + !EVP_CIPHER_CTX_ctrl(actx, EVP_CTRL_AEAD_SET_TAG, + (int)crypto_aead_taglen(cipher), NULL)) || + !EVP_EncryptInit_ex(actx, NULL, NULL, key, NULL)) { + EVP_CIPHER_CTX_free(actx); + return -1; + } + + aead_ctx->native_handle = actx; + + return 0; +} + +int ngtcp2_crypto_aead_ctx_decrypt_init(ngtcp2_crypto_aead_ctx *aead_ctx, + const ngtcp2_crypto_aead *aead, + const uint8_t *key, size_t noncelen) { + const EVP_CIPHER *cipher = aead->native_handle; + EVP_CIPHER_CTX *actx; + + actx = EVP_CIPHER_CTX_new(); + if (actx == NULL) { + return -1; + } + + if (!EVP_DecryptInit_ex(actx, cipher, NULL, NULL, NULL) || + !EVP_CIPHER_CTX_ctrl(actx, EVP_CTRL_AEAD_SET_IVLEN, (int)noncelen, + NULL) || + (cipher == EVP_aes_128_ccm() && + !EVP_CIPHER_CTX_ctrl(actx, EVP_CTRL_AEAD_SET_TAG, + (int)crypto_aead_taglen(cipher), NULL)) || + !EVP_DecryptInit_ex(actx, NULL, NULL, key, NULL)) { + EVP_CIPHER_CTX_free(actx); + return -1; + } + + aead_ctx->native_handle = actx; + + return 0; +} + +void ngtcp2_crypto_aead_ctx_free(ngtcp2_crypto_aead_ctx *aead_ctx) { + if (aead_ctx->native_handle) { + EVP_CIPHER_CTX_free(aead_ctx->native_handle); + } +} + +int ngtcp2_crypto_cipher_ctx_encrypt_init(ngtcp2_crypto_cipher_ctx *cipher_ctx, + const ngtcp2_crypto_cipher *cipher, + const uint8_t *key) { + EVP_CIPHER_CTX *actx; + + actx = EVP_CIPHER_CTX_new(); + if (actx == NULL) { + return -1; + } + + if (!EVP_EncryptInit_ex(actx, cipher->native_handle, NULL, key, NULL)) { + EVP_CIPHER_CTX_free(actx); + return -1; + } + + cipher_ctx->native_handle = actx; + + return 0; +} + +void ngtcp2_crypto_cipher_ctx_free(ngtcp2_crypto_cipher_ctx *cipher_ctx) { + if (cipher_ctx->native_handle) { + EVP_CIPHER_CTX_free(cipher_ctx->native_handle); + } +} + int ngtcp2_crypto_hkdf_extract(uint8_t *dest, const ngtcp2_crypto_md *md, const uint8_t *secret, size_t secretlen, const uint8_t *salt, size_t saltlen) { @@ -226,26 +312,18 @@ int ngtcp2_crypto_hkdf_expand(uint8_t *dest, size_t destlen, } int ngtcp2_crypto_encrypt(uint8_t *dest, const ngtcp2_crypto_aead *aead, + const ngtcp2_crypto_aead_ctx *aead_ctx, const uint8_t *plaintext, size_t plaintextlen, - const uint8_t *key, const uint8_t *nonce, - size_t noncelen, const uint8_t *ad, size_t adlen) { + const uint8_t *nonce, size_t noncelen, + const uint8_t *ad, size_t adlen) { const EVP_CIPHER *cipher = aead->native_handle; size_t taglen = crypto_aead_taglen(cipher); - EVP_CIPHER_CTX *actx; - int rv = 0; + EVP_CIPHER_CTX *actx = aead_ctx->native_handle; int len; - actx = EVP_CIPHER_CTX_new(); - if (actx == NULL) { - return -1; - } + (void)noncelen; - if (!EVP_EncryptInit_ex(actx, cipher, NULL, NULL, NULL) || - !EVP_CIPHER_CTX_ctrl(actx, EVP_CTRL_AEAD_SET_IVLEN, (int)noncelen, - NULL) || - (cipher == EVP_aes_128_ccm() && - !EVP_CIPHER_CTX_ctrl(actx, EVP_CTRL_AEAD_SET_TAG, (int)taglen, NULL)) || - !EVP_EncryptInit_ex(actx, NULL, NULL, key, nonce) || + if (!EVP_EncryptInit_ex(actx, NULL, NULL, NULL, nonce) || (cipher == EVP_aes_128_ccm() && !EVP_EncryptUpdate(actx, NULL, &len, NULL, (int)plaintextlen)) || !EVP_EncryptUpdate(actx, NULL, &len, ad, (int)adlen) || @@ -253,25 +331,25 @@ int ngtcp2_crypto_encrypt(uint8_t *dest, const ngtcp2_crypto_aead *aead, !EVP_EncryptFinal_ex(actx, dest + len, &len) || !EVP_CIPHER_CTX_ctrl(actx, EVP_CTRL_AEAD_GET_TAG, (int)taglen, dest + plaintextlen)) { - rv = -1; + return -1; } - EVP_CIPHER_CTX_free(actx); - - return rv; + return 0; } int ngtcp2_crypto_decrypt(uint8_t *dest, const ngtcp2_crypto_aead *aead, + const ngtcp2_crypto_aead_ctx *aead_ctx, const uint8_t *ciphertext, size_t ciphertextlen, - const uint8_t *key, const uint8_t *nonce, - size_t noncelen, const uint8_t *ad, size_t adlen) { + const uint8_t *nonce, size_t noncelen, + const uint8_t *ad, size_t adlen) { const EVP_CIPHER *cipher = aead->native_handle; size_t taglen = crypto_aead_taglen(cipher); - EVP_CIPHER_CTX *actx; - int rv = 0; + EVP_CIPHER_CTX *actx = aead_ctx->native_handle; int len; const uint8_t *tag; + (void)noncelen; + if (taglen > ciphertextlen) { return -1; } @@ -279,56 +357,37 @@ int ngtcp2_crypto_decrypt(uint8_t *dest, const ngtcp2_crypto_aead *aead, ciphertextlen -= taglen; tag = ciphertext + ciphertextlen; - actx = EVP_CIPHER_CTX_new(); - if (actx == NULL) { - return -1; - } - - if (!EVP_DecryptInit_ex(actx, cipher, NULL, NULL, NULL) || - !EVP_CIPHER_CTX_ctrl(actx, EVP_CTRL_AEAD_SET_IVLEN, (int)noncelen, - NULL) || - (cipher == EVP_aes_128_ccm() && - !EVP_CIPHER_CTX_ctrl(actx, EVP_CTRL_AEAD_SET_TAG, (int)taglen, - (uint8_t *)tag)) || - !EVP_DecryptInit_ex(actx, NULL, NULL, key, nonce) || + if (!EVP_DecryptInit_ex(actx, NULL, NULL, NULL, nonce) || + !EVP_CIPHER_CTX_ctrl(actx, EVP_CTRL_AEAD_SET_TAG, (int)taglen, + (uint8_t *)tag) || (cipher == EVP_aes_128_ccm() && !EVP_DecryptUpdate(actx, NULL, &len, NULL, (int)ciphertextlen)) || !EVP_DecryptUpdate(actx, NULL, &len, ad, (int)adlen) || !EVP_DecryptUpdate(actx, dest, &len, ciphertext, (int)ciphertextlen) || (cipher != EVP_aes_128_ccm() && - (!EVP_CIPHER_CTX_ctrl(actx, EVP_CTRL_AEAD_SET_TAG, (int)taglen, - (uint8_t *)tag) || - !EVP_DecryptFinal_ex(actx, dest + ciphertextlen, &len)))) { - rv = -1; + !EVP_DecryptFinal_ex(actx, dest + ciphertextlen, &len))) { + return -1; } - EVP_CIPHER_CTX_free(actx); - - return rv; + return 0; } int ngtcp2_crypto_hp_mask(uint8_t *dest, const ngtcp2_crypto_cipher *hp, - const uint8_t *hp_key, const uint8_t *sample) { + const ngtcp2_crypto_cipher_ctx *hp_ctx, + const uint8_t *sample) { static const uint8_t PLAINTEXT[] = "\x00\x00\x00\x00\x00"; - const EVP_CIPHER *cipher = hp->native_handle; - EVP_CIPHER_CTX *actx; - int rv = 0; + EVP_CIPHER_CTX *actx = hp_ctx->native_handle; int len; - actx = EVP_CIPHER_CTX_new(); - if (actx == NULL) { - return -1; - } + (void)hp; - if (!EVP_EncryptInit_ex(actx, cipher, NULL, hp_key, sample) || + if (!EVP_EncryptInit_ex(actx, NULL, NULL, NULL, sample) || !EVP_EncryptUpdate(actx, dest, &len, PLAINTEXT, sizeof(PLAINTEXT) - 1) || !EVP_EncryptFinal_ex(actx, dest + sizeof(PLAINTEXT) - 1, &len)) { - rv = -1; + return -1; } - EVP_CIPHER_CTX_free(actx); - - return rv; + return 0; } static OSSL_ENCRYPTION_LEVEL diff --git a/deps/ngtcp2/crypto/shared.c b/deps/ngtcp2/crypto/shared.c index bea088e85f3745..7e5219c52f2957 100644 --- a/deps/ngtcp2/crypto/shared.c +++ b/deps/ngtcp2/crypto/shared.c @@ -153,9 +153,11 @@ int ngtcp2_crypto_derive_and_install_rx_key(ngtcp2_conn *conn, uint8_t *key, const ngtcp2_crypto_ctx *ctx; const ngtcp2_crypto_aead *aead; const ngtcp2_crypto_md *md; + const ngtcp2_crypto_cipher *hp; + ngtcp2_crypto_aead_ctx aead_ctx = {0}; + ngtcp2_crypto_cipher_ctx hp_ctx = {0}; void *tls = ngtcp2_conn_get_tls_native_handle(conn); uint8_t keybuf[64], ivbuf[64], hp_keybuf[64]; - size_t keylen; size_t ivlen; int rv; @@ -185,7 +187,7 @@ int ngtcp2_crypto_derive_and_install_rx_key(ngtcp2_conn *conn, uint8_t *key, aead = &ctx->aead; md = &ctx->md; - keylen = ngtcp2_crypto_aead_keylen(aead); + hp = &ctx->hp; ivlen = ngtcp2_crypto_packet_protection_ivlen(aead); if (ngtcp2_crypto_derive_packet_protection_key(key, iv, hp_key, aead, md, @@ -193,36 +195,79 @@ int ngtcp2_crypto_derive_and_install_rx_key(ngtcp2_conn *conn, uint8_t *key, return -1; } + if (ngtcp2_crypto_aead_ctx_decrypt_init(&aead_ctx, aead, key, ivlen) != 0) { + goto fail; + } + + if (ngtcp2_crypto_cipher_ctx_encrypt_init(&hp_ctx, hp, hp_key) != 0) { + goto fail; + } + switch (level) { case NGTCP2_CRYPTO_LEVEL_EARLY: - rv = ngtcp2_conn_install_early_key(conn, key, iv, hp_key, keylen, ivlen); + rv = ngtcp2_conn_install_early_key(conn, &aead_ctx, iv, ivlen, &hp_ctx); if (rv != 0) { - return -1; + goto fail; } break; case NGTCP2_CRYPTO_LEVEL_HANDSHAKE: - rv = ngtcp2_conn_install_rx_handshake_key(conn, key, iv, hp_key, keylen, - ivlen); + rv = ngtcp2_conn_install_rx_handshake_key(conn, &aead_ctx, iv, ivlen, + &hp_ctx); if (rv != 0) { - return -1; + goto fail; } break; case NGTCP2_CRYPTO_LEVEL_APP: if (!ngtcp2_conn_is_server(conn)) { rv = ngtcp2_crypto_set_remote_transport_params(conn, tls); if (rv != 0) { - return -1; + goto fail; } } - rv = ngtcp2_conn_install_rx_key(conn, secret, key, iv, hp_key, secretlen, - keylen, ivlen); + rv = ngtcp2_conn_install_rx_key(conn, secret, secretlen, &aead_ctx, iv, + ivlen, &hp_ctx); if (rv != 0) { - return -1; + goto fail; } break; default: + goto fail; + } + + return 0; + +fail: + ngtcp2_crypto_cipher_ctx_free(&hp_ctx); + ngtcp2_crypto_aead_ctx_free(&aead_ctx); + + return -1; +} + +/* + * crypto_set_local_transport_params gets local QUIC transport + * parameters from |conn| and sets it to |tls|. + * + * This function returns 0 if it succeeds, or -1. + */ +static int crypto_set_local_transport_params(ngtcp2_conn *conn, void *tls) { + ngtcp2_transport_params_type exttype = + ngtcp2_conn_is_server(conn) + ? NGTCP2_TRANSPORT_PARAMS_TYPE_ENCRYPTED_EXTENSIONS + : NGTCP2_TRANSPORT_PARAMS_TYPE_CLIENT_HELLO; + ngtcp2_transport_params params; + ngtcp2_ssize nwrite; + uint8_t buf[256]; + + ngtcp2_conn_get_local_transport_params(conn, ¶ms); + + nwrite = ngtcp2_encode_transport_params(buf, sizeof(buf), exttype, ¶ms); + if (nwrite < 0) { + return -1; + } + + if (ngtcp2_crypto_set_local_transport_params(tls, buf, (size_t)nwrite) != 0) { return -1; } @@ -237,9 +282,11 @@ int ngtcp2_crypto_derive_and_install_tx_key(ngtcp2_conn *conn, uint8_t *key, const ngtcp2_crypto_ctx *ctx; const ngtcp2_crypto_aead *aead; const ngtcp2_crypto_md *md; + const ngtcp2_crypto_cipher *hp; + ngtcp2_crypto_aead_ctx aead_ctx = {0}; + ngtcp2_crypto_cipher_ctx hp_ctx = {0}; void *tls = ngtcp2_conn_get_tls_native_handle(conn); uint8_t keybuf[64], ivbuf[64], hp_keybuf[64]; - size_t keylen; size_t ivlen; int rv; @@ -269,7 +316,7 @@ int ngtcp2_crypto_derive_and_install_tx_key(ngtcp2_conn *conn, uint8_t *key, aead = &ctx->aead; md = &ctx->md; - keylen = ngtcp2_crypto_aead_keylen(aead); + hp = &ctx->hp; ivlen = ngtcp2_crypto_packet_protection_ivlen(aead); if (ngtcp2_crypto_derive_packet_protection_key(key, iv, hp_key, aead, md, @@ -277,40 +324,59 @@ int ngtcp2_crypto_derive_and_install_tx_key(ngtcp2_conn *conn, uint8_t *key, return -1; } + if (ngtcp2_crypto_aead_ctx_encrypt_init(&aead_ctx, aead, key, ivlen) != 0) { + goto fail; + } + + if (ngtcp2_crypto_cipher_ctx_encrypt_init(&hp_ctx, hp, hp_key) != 0) { + goto fail; + } + switch (level) { case NGTCP2_CRYPTO_LEVEL_EARLY: - rv = ngtcp2_conn_install_early_key(conn, key, iv, hp_key, keylen, ivlen); + rv = ngtcp2_conn_install_early_key(conn, &aead_ctx, iv, ivlen, &hp_ctx); if (rv != 0) { - return -1; + goto fail; } break; case NGTCP2_CRYPTO_LEVEL_HANDSHAKE: + rv = ngtcp2_conn_install_tx_handshake_key(conn, &aead_ctx, iv, ivlen, + &hp_ctx); + if (rv != 0) { + goto fail; + } + if (ngtcp2_conn_is_server(conn)) { rv = ngtcp2_crypto_set_remote_transport_params(conn, tls); if (rv != 0) { - return -1; + goto fail; } - } - rv = ngtcp2_conn_install_tx_handshake_key(conn, key, iv, hp_key, keylen, - ivlen); - if (rv != 0) { - return -1; + if (crypto_set_local_transport_params(conn, tls) != 0) { + goto fail; + } } + break; case NGTCP2_CRYPTO_LEVEL_APP: - rv = ngtcp2_conn_install_tx_key(conn, secret, key, iv, hp_key, secretlen, - keylen, ivlen); + rv = ngtcp2_conn_install_tx_key(conn, secret, secretlen, &aead_ctx, iv, + ivlen, &hp_ctx); if (rv != 0) { - return -1; + goto fail; } break; default: - return -1; + goto fail; } return 0; + +fail: + ngtcp2_crypto_cipher_ctx_free(&hp_ctx); + ngtcp2_crypto_aead_ctx_free(&aead_ctx); + + return -1; } int ngtcp2_crypto_derive_and_install_initial_key( @@ -329,7 +395,13 @@ int ngtcp2_crypto_derive_and_install_initial_key( uint8_t tx_hp_keybuf[NGTCP2_CRYPTO_INITIAL_KEYLEN]; ngtcp2_crypto_ctx ctx; ngtcp2_crypto_aead retry_aead; + ngtcp2_crypto_aead_ctx rx_aead_ctx = {0}; + ngtcp2_crypto_cipher_ctx rx_hp_ctx = {0}; + ngtcp2_crypto_aead_ctx tx_aead_ctx = {0}; + ngtcp2_crypto_cipher_ctx tx_hp_ctx = {0}; + ngtcp2_crypto_aead_ctx retry_aead_ctx = {0}; int rv; + int server = ngtcp2_conn_is_server(conn); ngtcp2_crypto_ctx_initial(&ctx); @@ -366,8 +438,8 @@ int ngtcp2_crypto_derive_and_install_initial_key( if (ngtcp2_crypto_derive_initial_secrets( rx_secret, tx_secret, initial_secret, client_dcid, - ngtcp2_conn_is_server(conn) ? NGTCP2_CRYPTO_SIDE_SERVER - : NGTCP2_CRYPTO_SIDE_CLIENT) != 0) { + server ? NGTCP2_CRYPTO_SIDE_SERVER : NGTCP2_CRYPTO_SIDE_CLIENT) != + 0) { return -1; } @@ -383,27 +455,69 @@ int ngtcp2_crypto_derive_and_install_initial_key( return -1; } - rv = ngtcp2_conn_install_initial_key( - conn, rx_key, rx_iv, rx_hp_key, tx_key, tx_iv, tx_hp_key, - NGTCP2_CRYPTO_INITIAL_KEYLEN, NGTCP2_CRYPTO_INITIAL_IVLEN); + if (ngtcp2_crypto_aead_ctx_decrypt_init(&rx_aead_ctx, &ctx.aead, rx_key, + NGTCP2_CRYPTO_INITIAL_IVLEN) != 0) { + goto fail; + } + + if (ngtcp2_crypto_cipher_ctx_encrypt_init(&rx_hp_ctx, &ctx.hp, rx_hp_key) != + 0) { + goto fail; + } + + if (ngtcp2_crypto_aead_ctx_encrypt_init(&tx_aead_ctx, &ctx.aead, tx_key, + NGTCP2_CRYPTO_INITIAL_IVLEN) != 0) { + goto fail; + } + + if (ngtcp2_crypto_cipher_ctx_encrypt_init(&tx_hp_ctx, &ctx.hp, tx_hp_key) != + 0) { + goto fail; + } + + if (!server && !ngtcp2_conn_after_retry(conn)) { + ngtcp2_crypto_aead_retry(&retry_aead); + + if (ngtcp2_crypto_aead_ctx_encrypt_init( + &retry_aead_ctx, &retry_aead, (const uint8_t *)NGTCP2_RETRY_KEY, + sizeof(NGTCP2_RETRY_NONCE) - 1) != 0) { + goto fail; + } + } + + rv = ngtcp2_conn_install_initial_key(conn, &rx_aead_ctx, rx_iv, &rx_hp_ctx, + &tx_aead_ctx, tx_iv, &tx_hp_ctx, + NGTCP2_CRYPTO_INITIAL_IVLEN); if (rv != 0) { - return -1; + goto fail; } - ngtcp2_conn_set_retry_aead(conn, ngtcp2_crypto_aead_retry(&retry_aead)); + if (retry_aead_ctx.native_handle) { + ngtcp2_conn_set_retry_aead(conn, &retry_aead, &retry_aead_ctx); + } return 0; + +fail: + ngtcp2_crypto_aead_ctx_free(&retry_aead_ctx); + ngtcp2_crypto_cipher_ctx_free(&tx_hp_ctx); + ngtcp2_crypto_aead_ctx_free(&tx_aead_ctx); + ngtcp2_crypto_cipher_ctx_free(&rx_hp_ctx); + ngtcp2_crypto_aead_ctx_free(&rx_aead_ctx); + + return -1; } -int ngtcp2_crypto_update_key(ngtcp2_conn *conn, uint8_t *rx_secret, - uint8_t *tx_secret, uint8_t *rx_key, - uint8_t *rx_iv, uint8_t *tx_key, uint8_t *tx_iv, - const uint8_t *current_rx_secret, - const uint8_t *current_tx_secret, - size_t secretlen) { +int ngtcp2_crypto_update_key( + ngtcp2_conn *conn, uint8_t *rx_secret, uint8_t *tx_secret, + ngtcp2_crypto_aead_ctx *rx_aead_ctx, uint8_t *rx_key, uint8_t *rx_iv, + ngtcp2_crypto_aead_ctx *tx_aead_ctx, uint8_t *tx_key, uint8_t *tx_iv, + const uint8_t *current_rx_secret, const uint8_t *current_tx_secret, + size_t secretlen) { const ngtcp2_crypto_ctx *ctx = ngtcp2_conn_get_crypto_ctx(conn); const ngtcp2_crypto_aead *aead = &ctx->aead; const ngtcp2_crypto_md *md = &ctx->md; + size_t ivlen = ngtcp2_crypto_packet_protection_ivlen(aead); if (ngtcp2_crypto_update_traffic_secret(rx_secret, md, current_rx_secret, secretlen) != 0) { @@ -425,51 +539,69 @@ int ngtcp2_crypto_update_key(ngtcp2_conn *conn, uint8_t *rx_secret, return -1; } + if (ngtcp2_crypto_aead_ctx_decrypt_init(rx_aead_ctx, aead, rx_key, ivlen) != + 0) { + return -1; + } + + if (ngtcp2_crypto_aead_ctx_encrypt_init(tx_aead_ctx, aead, tx_key, ivlen) != + 0) { + ngtcp2_crypto_aead_ctx_free(rx_aead_ctx); + rx_aead_ctx->native_handle = NULL; + return -1; + } + return 0; } int ngtcp2_crypto_encrypt_cb(uint8_t *dest, const ngtcp2_crypto_aead *aead, + const ngtcp2_crypto_aead_ctx *aead_ctx, const uint8_t *plaintext, size_t plaintextlen, - const uint8_t *key, const uint8_t *nonce, - size_t noncelen, const uint8_t *ad, size_t adlen) { - if (ngtcp2_crypto_encrypt(dest, aead, plaintext, plaintextlen, key, nonce, - noncelen, ad, adlen) != 0) { + const uint8_t *nonce, size_t noncelen, + const uint8_t *ad, size_t adlen) { + if (ngtcp2_crypto_encrypt(dest, aead, aead_ctx, plaintext, plaintextlen, + nonce, noncelen, ad, adlen) != 0) { return NGTCP2_ERR_CALLBACK_FAILURE; } return 0; } int ngtcp2_crypto_decrypt_cb(uint8_t *dest, const ngtcp2_crypto_aead *aead, + const ngtcp2_crypto_aead_ctx *aead_ctx, const uint8_t *ciphertext, size_t ciphertextlen, - const uint8_t *key, const uint8_t *nonce, - size_t noncelen, const uint8_t *ad, size_t adlen) { - if (ngtcp2_crypto_decrypt(dest, aead, ciphertext, ciphertextlen, key, nonce, - noncelen, ad, adlen) != 0) { + const uint8_t *nonce, size_t noncelen, + const uint8_t *ad, size_t adlen) { + if (ngtcp2_crypto_decrypt(dest, aead, aead_ctx, ciphertext, ciphertextlen, + nonce, noncelen, ad, adlen) != 0) { return NGTCP2_ERR_TLS_DECRYPT; } return 0; } int ngtcp2_crypto_hp_mask_cb(uint8_t *dest, const ngtcp2_crypto_cipher *hp, - const uint8_t *hp_key, const uint8_t *sample) { - if (ngtcp2_crypto_hp_mask(dest, hp, hp_key, sample) != 0) { + const ngtcp2_crypto_cipher_ctx *hp_ctx, + const uint8_t *sample) { + if (ngtcp2_crypto_hp_mask(dest, hp, hp_ctx, sample) != 0) { return NGTCP2_ERR_CALLBACK_FAILURE; } return 0; } -int ngtcp2_crypto_update_key_cb(ngtcp2_conn *conn, uint8_t *rx_secret, - uint8_t *tx_secret, uint8_t *rx_key, - uint8_t *rx_iv, uint8_t *tx_key, uint8_t *tx_iv, - const uint8_t *current_rx_secret, - const uint8_t *current_tx_secret, - size_t secretlen, void *user_data) { +int ngtcp2_crypto_update_key_cb( + ngtcp2_conn *conn, uint8_t *rx_secret, uint8_t *tx_secret, + ngtcp2_crypto_aead_ctx *rx_aead_ctx, uint8_t *rx_iv, + ngtcp2_crypto_aead_ctx *tx_aead_ctx, uint8_t *tx_iv, + const uint8_t *current_rx_secret, const uint8_t *current_tx_secret, + size_t secretlen, void *user_data) { + uint8_t rx_key[64]; + uint8_t tx_key[64]; (void)conn; (void)user_data; - if (ngtcp2_crypto_update_key(conn, rx_secret, tx_secret, rx_key, rx_iv, - tx_key, tx_iv, current_rx_secret, - current_tx_secret, secretlen) != 0) { + if (ngtcp2_crypto_update_key(conn, rx_secret, tx_secret, rx_aead_ctx, rx_key, + rx_iv, tx_aead_ctx, tx_key, tx_iv, + current_rx_secret, current_tx_secret, + secretlen) != 0) { return NGTCP2_ERR_CALLBACK_FAILURE; } return 0; @@ -509,6 +641,8 @@ ngtcp2_ssize ngtcp2_crypto_write_connection_close(uint8_t *dest, size_t destlen, uint8_t tx_hp_key[NGTCP2_CRYPTO_INITIAL_KEYLEN]; ngtcp2_crypto_ctx ctx; ngtcp2_ssize spktlen; + ngtcp2_crypto_aead_ctx aead_ctx = {0}; + ngtcp2_crypto_cipher_ctx hp_ctx = {0}; ngtcp2_crypto_ctx_initial(&ctx); @@ -524,13 +658,28 @@ ngtcp2_ssize ngtcp2_crypto_write_connection_close(uint8_t *dest, size_t destlen, return -1; } + if (ngtcp2_crypto_aead_ctx_encrypt_init(&aead_ctx, &ctx.aead, tx_key, + NGTCP2_CRYPTO_INITIAL_IVLEN) != 0) { + spktlen = -1; + goto end; + } + + if (ngtcp2_crypto_cipher_ctx_encrypt_init(&hp_ctx, &ctx.hp, tx_hp_key) != 0) { + spktlen = -1; + goto end; + } + spktlen = ngtcp2_pkt_write_connection_close( dest, destlen, dcid, scid, error_code, ngtcp2_crypto_encrypt_cb, - &ctx.aead, tx_key, tx_iv, ngtcp2_crypto_hp_mask_cb, &ctx.hp, tx_hp_key); + &ctx.aead, &aead_ctx, tx_iv, ngtcp2_crypto_hp_mask_cb, &ctx.hp, &hp_ctx); if (spktlen < 0) { - return -1; + spktlen = -1; } +end: + ngtcp2_crypto_cipher_ctx_free(&hp_ctx); + ngtcp2_crypto_aead_ctx_free(&aead_ctx); + return spktlen; } @@ -541,45 +690,26 @@ ngtcp2_ssize ngtcp2_crypto_write_retry(uint8_t *dest, size_t destlen, const uint8_t *token, size_t tokenlen) { ngtcp2_crypto_aead aead; ngtcp2_ssize spktlen; + ngtcp2_crypto_aead_ctx aead_ctx = {0}; ngtcp2_crypto_aead_retry(&aead); - spktlen = ngtcp2_pkt_write_retry(dest, destlen, dcid, scid, odcid, token, - tokenlen, ngtcp2_crypto_encrypt_cb, &aead); - if (spktlen < 0) { + if (ngtcp2_crypto_aead_ctx_encrypt_init(&aead_ctx, &aead, + (const uint8_t *)NGTCP2_RETRY_KEY, + NGTCP2_CRYPTO_INITIAL_IVLEN) != 0) { return -1; } - return spktlen; -} - -/* - * crypto_set_local_transport_params gets local QUIC transport - * parameters from |conn| and sets it to |tls|. - * - * This function returns 0 if it succeeds, or -1. - */ -static int crypto_set_local_transport_params(ngtcp2_conn *conn, void *tls) { - ngtcp2_transport_params_type exttype = - ngtcp2_conn_is_server(conn) - ? NGTCP2_TRANSPORT_PARAMS_TYPE_ENCRYPTED_EXTENSIONS - : NGTCP2_TRANSPORT_PARAMS_TYPE_CLIENT_HELLO; - ngtcp2_transport_params params; - ngtcp2_ssize nwrite; - uint8_t buf[256]; - - ngtcp2_conn_get_local_transport_params(conn, ¶ms); - - nwrite = ngtcp2_encode_transport_params(buf, sizeof(buf), exttype, ¶ms); - if (nwrite < 0) { - return -1; + spktlen = + ngtcp2_pkt_write_retry(dest, destlen, dcid, scid, odcid, token, tokenlen, + ngtcp2_crypto_encrypt_cb, &aead, &aead_ctx); + if (spktlen < 0) { + spktlen = -1; } - if (ngtcp2_crypto_set_local_transport_params(tls, buf, (size_t)nwrite) != 0) { - return -1; - } + ngtcp2_crypto_aead_ctx_free(&aead_ctx); - return 0; + return spktlen; } /* @@ -588,25 +718,23 @@ static int crypto_set_local_transport_params(ngtcp2_conn *conn, void *tls) { */ static int crypto_setup_initial_crypto(ngtcp2_conn *conn, const ngtcp2_cid *dcid) { - void *tls = ngtcp2_conn_get_tls_native_handle(conn); - - if (ngtcp2_crypto_derive_and_install_initial_key(conn, NULL, NULL, NULL, NULL, - NULL, NULL, NULL, NULL, NULL, - dcid) != 0) { - return -1; - } - - return crypto_set_local_transport_params(conn, tls); + return ngtcp2_crypto_derive_and_install_initial_key( + conn, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, dcid); } int ngtcp2_crypto_client_initial_cb(ngtcp2_conn *conn, void *user_data) { const ngtcp2_cid *dcid = ngtcp2_conn_get_dcid(conn); + void *tls = ngtcp2_conn_get_tls_native_handle(conn); (void)user_data; if (crypto_setup_initial_crypto(conn, dcid) != 0) { return NGTCP2_ERR_CALLBACK_FAILURE; } + if (crypto_set_local_transport_params(conn, tls) != 0) { + return NGTCP2_ERR_CALLBACK_FAILURE; + } + if (ngtcp2_crypto_read_write_crypto_data(conn, NGTCP2_CRYPTO_LEVEL_INITIAL, NULL, 0) != 0) { return NGTCP2_ERR_CALLBACK_FAILURE; @@ -639,3 +767,20 @@ int ngtcp2_crypto_recv_client_initial_cb(ngtcp2_conn *conn, return 0; } + +void ngtcp2_crypto_delete_crypto_aead_ctx_cb(ngtcp2_conn *conn, + ngtcp2_crypto_aead_ctx *aead_ctx, + void *user_data) { + (void)conn; + (void)user_data; + + ngtcp2_crypto_aead_ctx_free(aead_ctx); +} + +void ngtcp2_crypto_delete_crypto_cipher_ctx_cb( + ngtcp2_conn *conn, ngtcp2_crypto_cipher_ctx *cipher_ctx, void *user_data) { + (void)conn; + (void)user_data; + + ngtcp2_crypto_cipher_ctx_free(cipher_ctx); +} diff --git a/deps/ngtcp2/crypto/shared.h b/deps/ngtcp2/crypto/shared.h index a904f6994fb2fa..b70513afb3a4b0 100644 --- a/deps/ngtcp2/crypto/shared.h +++ b/deps/ngtcp2/crypto/shared.h @@ -110,7 +110,6 @@ int ngtcp2_crypto_set_local_transport_params(void *tls, const uint8_t *buf, */ int ngtcp2_crypto_set_remote_transport_params(ngtcp2_conn *conn, void *tls); - /** * @function * @@ -165,4 +164,26 @@ int ngtcp2_crypto_derive_and_install_initial_key( uint8_t *tx_key, uint8_t *tx_iv, uint8_t *tx_hp, const ngtcp2_cid *client_dcid); +/** + * @function + * + * `ngtcp2_crypto_cipher_ctx_encrypt_init` initializes |cipher_ctx| + * with new cipher context object for encryption which is constructed + * to use |key| as encryption key. |cipher| specifies cipher to use. + * + * This function returns 0 if it succeeds, or -1. + */ +int ngtcp2_crypto_cipher_ctx_encrypt_init(ngtcp2_crypto_cipher_ctx *cipher_ctx, + const ngtcp2_crypto_cipher *cipher, + const uint8_t *key); + +/** + * @function + * + * `ngtcp2_crypto_cipher_ctx_free` frees up resources used by + * |cipher_ctx|. This function does not free the memory pointed by + * |cipher_ctx| itself. + */ +void ngtcp2_crypto_cipher_ctx_free(ngtcp2_crypto_cipher_ctx *cipher_ctx); + #endif /* NGTCP2_SHARED_H */ diff --git a/deps/ngtcp2/lib/includes/ngtcp2/ngtcp2.h b/deps/ngtcp2/lib/includes/ngtcp2/ngtcp2.h index 29cf3e542509d2..3373adb9f91a52 100644 --- a/deps/ngtcp2/lib/includes/ngtcp2/ngtcp2.h +++ b/deps/ngtcp2/lib/includes/ngtcp2/ngtcp2.h @@ -49,6 +49,12 @@ extern "C" { #include #include +#ifdef WIN32 +# include +#else +# include +#endif + #include #ifdef NGTCP2_STATICLIB @@ -190,6 +196,15 @@ typedef struct ngtcp2_mem { "\xaf\xbf\xec\x28\x99\x93\xd2\x4c\x9e\x97\x86\xf1\x9c\x61\x11\xe0\x43\x90" \ "\xa8\x99" +/* NGTCP2_RETRY_KEY is an encryption key to create integrity tag of + Retry packet. */ +#define NGTCP2_RETRY_KEY \ + "\xcc\xce\x18\x7e\xd0\x9a\x09\xd0\x57\x28\x15\x5a\x6c\xb9\x6b\xe1" + +/* NGTCP2_RETRY_NONCE is nonce used when generating integrity tag of + Retry packet. */ +#define NGTCP2_RETRY_NONCE "\xe5\x49\x30\xf9\x7f\x21\x36\xf0\x53\x0a\x8c\x1c" + /* NGTCP2_HP_MASKLEN is the length of header protection mask. */ #define NGTCP2_HP_MASKLEN 5 @@ -212,6 +227,9 @@ typedef struct ngtcp2_mem { nanosecond. */ #define NGTCP2_NANOSECONDS ((uint64_t)1ULL) +/* NGTCP2_DEFAULT_INITIAL_RTT is a default initial RTT. */ +#define NGTCP2_DEFAULT_INITIAL_RTT (333 * NGTCP2_MILLISECONDS) + #if defined(__cplusplus) && __cplusplus >= 201103L typedef enum ngtcp2_lib_error : int { #else @@ -248,7 +266,7 @@ typedef enum ngtcp2_lib_error { NGTCP2_ERR_CONN_ID_BLOCKED = -237, NGTCP2_ERR_INTERNAL = -238, NGTCP2_ERR_CRYPTO_BUFFER_EXCEEDED = -239, - NGTCP2_ERR_WRITE_STREAM_MORE = -240, + NGTCP2_ERR_WRITE_MORE = -240, NGTCP2_ERR_RETRY = -241, NGTCP2_ERR_DROP_CONN = -242, NGTCP2_ERR_FATAL = -500, @@ -488,9 +506,8 @@ typedef struct ngtcp2_transport_params { ngtcp2_preferred_addr preferred_address; /* original_dcid is the Destination Connection ID field from the first Initial packet from client. Server must specify this - field. If application specifies retry_scid_present to nonzero, - then it must also specify this field. It is expected that - application knows the original Destination Connection ID, for + field. It is expected that application knows the original + Destination Connection ID even if it sends Retry packet, for example, by including it in retry token. Otherwise, application should not specify this field. */ ngtcp2_cid original_dcid; @@ -573,6 +590,7 @@ typedef struct ngtcp2_conn_stat { ngtcp2_duration min_rtt; ngtcp2_duration smoothed_rtt; ngtcp2_duration rttvar; + ngtcp2_duration initial_rtt; size_t pto_count; ngtcp2_tstamp loss_detection_timer; /* last_tx_pkt_ts corresponds to @@ -678,8 +696,30 @@ typedef struct ngtcp2_cc { ngtcp2_conn_server_new. */ typedef void (*ngtcp2_printf)(void *user_data, const char *format, ...); -typedef void (*ngtcp2_qlog_write)(void *user_data, const void *data, - size_t datalen); +/** + * @enum + * + * :type:`ngtcp2_qlog_write_flag` defines the set of flags passed to + * :type:`ngtcp2_qlog_write` callback. + */ +typedef enum ngtcp2_qlog_write_flag { + NGTCP2_QLOG_WRITE_FLAG_NONE = 0, + /** + * NGTCP2_QLOG_WRITE_FLAG_FIN indicates that this is the final call + * to :type:`ngtcp2_qlog_write` in the current connection. + */ + NGTCP2_QLOG_WRITE_FLAG_FIN = 0x01 +} ngtcp2_qlog_write_flag; + +/** + * @functypedef + * + * :type:`ngtcp2_qlog_write` is a callback function which is called to + * write qlog |data| of length |datalen| bytes. |flags| is bitwise OR + * of zero or more of :type:`ngtcp2_qlog_write_flag`. + */ +typedef void (*ngtcp2_qlog_write)(void *user_data, uint32_t flags, + const void *data, size_t datalen); typedef struct ngtcp2_qlog_settings { /* odcid is Original Destination Connection ID sent by client. It @@ -699,6 +739,8 @@ typedef struct ngtcp2_settings { ngtcp2_cc *cc; /* initial_ts is an initial timestamp given to the library. */ ngtcp2_tstamp initial_ts; + /* initial_rtt is an initial RTT. */ + ngtcp2_duration initial_rtt; /* log_printf is a function that the library uses to write logs. NULL means no logging output. */ ngtcp2_printf log_printf; @@ -730,9 +772,9 @@ typedef struct ngtcp2_settings { typedef struct ngtcp2_addr { /* addrlen is the length of addr. */ size_t addrlen; - /* addr points to the buffer which contains endpoint address. It is - opaque to the ngtcp2 library. It must not be NULL. */ - uint8_t *addr; + /* addr points to the buffer which contains endpoint address. It + must not be NULL. */ + struct sockaddr *addr; /* user_data is an arbitrary data and opaque to the library. */ void *user_data; } ngtcp2_addr; @@ -757,8 +799,8 @@ typedef struct ngtcp2_path { * the longest addresses. */ typedef struct ngtcp2_path_storage { - uint8_t local_addrbuf[128]; - uint8_t remote_addrbuf[128]; + struct sockaddr_storage local_addrbuf; + struct sockaddr_storage remote_addrbuf; ngtcp2_path path; } ngtcp2_path_storage; @@ -799,6 +841,30 @@ typedef struct ngtcp2_crypto_cipher { void *native_handle; } ngtcp2_crypto_cipher; +/** + * @struct + * + * `ngtcp2_crypto_aead_ctx` is a wrapper around native AEAD cipher + * context object. It should be initialized with a specific key. + * ngtcp2 library reuses this context object to encrypt or decrypt + * multiple packets. + */ +typedef struct ngtcp2_crypto_aead_ctx { + void *native_handle; +} ngtcp2_crypto_aead_ctx; + +/** + * @struct + * + * `ngtcp2_crypto_cipher_ctx` is a wrapper around native cipher + * context object. It should be initialized with a specific key. + * ngtcp2 library reuses this context object to encrypt or decrypt + * multiple packet headers. + */ +typedef struct ngtcp2_crypto_cipher_ctx { + void *native_handle; +} ngtcp2_crypto_cipher_ctx; + /** * @function * @@ -1200,9 +1266,10 @@ typedef int (*ngtcp2_recv_retry)(ngtcp2_conn *conn, const ngtcp2_pkt_hd *hd, * :type:`ngtcp2_encrypt` is invoked when the ngtcp2 library asks the * application to encrypt packet payload. The packet payload to * encrypt is passed as |plaintext| of length |plaintextlen|. The - * encryption key is passed as |key|. The nonce is passed as |nonce| - * of length |noncelen|. The ad, Additional Data to AEAD, is passed - * as |ad| of length |adlen|. + * AEAD cipher is |aead|. |aead_ctx| is the AEAD cipher context + * object which is initialized with encryption key. The nonce is + * passed as |nonce| of length |noncelen|. The ad, Additional Data to + * AEAD, is passed as |ad| of length |adlen|. * * The implementation of this callback must encrypt |plaintext| using * the negotiated cipher suite and write the ciphertext into the @@ -1216,9 +1283,10 @@ typedef int (*ngtcp2_recv_retry)(ngtcp2_conn *conn, const ngtcp2_pkt_hd *hd, * return immediately. */ typedef int (*ngtcp2_encrypt)(uint8_t *dest, const ngtcp2_crypto_aead *aead, + const ngtcp2_crypto_aead_ctx *aead_ctx, const uint8_t *plaintext, size_t plaintextlen, - const uint8_t *key, const uint8_t *nonce, - size_t noncelen, const uint8_t *ad, size_t adlen); + const uint8_t *nonce, size_t noncelen, + const uint8_t *ad, size_t adlen); /** * @functypedef @@ -1226,7 +1294,8 @@ typedef int (*ngtcp2_encrypt)(uint8_t *dest, const ngtcp2_crypto_aead *aead, * :type:`ngtcp2_decrypt` is invoked when the ngtcp2 library asks the * application to decrypt packet payload. The packet payload to * decrypt is passed as |ciphertext| of length |ciphertextlen|. The - * decryption key is passed as |key| of length |keylen|. The nonce is + * AEAD cipher is |aead|. |aead_ctx| is the AEAD cipher context + * object which is initialized with decryption key. The nonce is * passed as |nonce| of length |noncelen|. The ad, Additional Data to * AEAD, is passed as |ad| of length |adlen|. * @@ -1243,16 +1312,19 @@ typedef int (*ngtcp2_encrypt)(uint8_t *dest, const ngtcp2_crypto_aead *aead, * makes the library call return immediately. */ typedef int (*ngtcp2_decrypt)(uint8_t *dest, const ngtcp2_crypto_aead *aead, + const ngtcp2_crypto_aead_ctx *aead_ctx, const uint8_t *ciphertext, size_t ciphertextlen, - const uint8_t *key, const uint8_t *nonce, - size_t noncelen, const uint8_t *ad, size_t adlen); + const uint8_t *nonce, size_t noncelen, + const uint8_t *ad, size_t adlen); /** * @functypedef * * :type:`ngtcp2_hp_mask` is invoked when the ngtcp2 library asks the * application to produce mask to encrypt or decrypt packet header. - * The key is passed as |hp_key|. The sample is passed as |sample|. + * The encryption cipher is |hp|. |hp_ctx| is the cipher context + * object which is initialized with header protection key. The sample + * is passed as |sample|. * * The implementation of this callback must produce a mask using the * header protection cipher suite specified by QUIC specification and @@ -1265,7 +1337,8 @@ typedef int (*ngtcp2_decrypt)(uint8_t *dest, const ngtcp2_crypto_aead *aead, * return immediately. */ typedef int (*ngtcp2_hp_mask)(uint8_t *dest, const ngtcp2_crypto_cipher *hp, - const uint8_t *hp_key, const uint8_t *sample); + const ngtcp2_crypto_cipher_ctx *hp_ctx, + const uint8_t *sample); /** * @enum @@ -1457,8 +1530,7 @@ typedef int (*ngtcp2_extend_max_stream_data)(ngtcp2_conn *conn, * :enum:`NGTCP2_ERR_CALLBACK_FAILURE` makes the library call return * immediately. */ -typedef int (*ngtcp2_rand)(ngtcp2_conn *conn, uint8_t *dest, size_t destlen, - ngtcp2_rand_ctx ctx, void *user_data); +typedef int (*ngtcp2_rand)(uint8_t *dest, size_t destlen, ngtcp2_rand_ctx ctx); /** * @functypedef @@ -1500,28 +1572,31 @@ typedef int (*ngtcp2_remove_connection_id)(ngtcp2_conn *conn, * * :type:`ngtcp2_update_key` is a callback function which tells the * application that it must generate new packet protection keying - * materials. The current set of secrets are given as - * |current_rx_secret| and |current_tx_secret| of length |secretlen|. - * They are decryption and encryption secrets respectively. + * materials and AEAD cipher context objects with new keys. The + * current set of secrets are given as |current_rx_secret| and + * |current_tx_secret| of length |secretlen|. They are decryption and + * encryption secrets respectively. * * The application has to generate new secrets and keys for both - * encryption and decryption, and write decryption secret, key and IV - * to the buffer pointed by |rx_secret|, |rx_key| and |rx_iv| - * respectively. Similarly, write encryption secret, key and IV to - * the buffer pointed by |tx_secret|, |tx_key| and |tx_iv|. All given + * encryption and decryption, and write decryption secret and IV to + * the buffer pointed by |rx_secret| and |rx_iv| respectively. It + * also has to create new AEAD cipher context object with new + * decryption key and initialize |rx_aead_ctx| with it. Similarly, + * write encryption secret and IV to the buffer pointed by |tx_secret| + * and |tx_iv|. Create new AEAD cipher context object with new + * encryption key and initialize |tx_aead_ctx| with it. All given * buffers have the enough capacity to store secret, key and IV. * * The callback function must return 0 if it succeeds. Returning * :enum:`NGTCP2_ERR_CALLBACK_FAILURE` makes the library call return * immediately. */ -typedef int (*ngtcp2_update_key)(ngtcp2_conn *conn, uint8_t *rx_secret, - uint8_t *tx_secret, uint8_t *rx_key, - uint8_t *rx_iv, uint8_t *tx_key, - uint8_t *tx_iv, - const uint8_t *current_rx_secret, - const uint8_t *current_tx_secret, - size_t secretlen, void *user_data); +typedef int (*ngtcp2_update_key)( + ngtcp2_conn *conn, uint8_t *rx_secret, uint8_t *tx_secret, + ngtcp2_crypto_aead_ctx *rx_aead_ctx, uint8_t *rx_iv, + ngtcp2_crypto_aead_ctx *tx_aead_ctx, uint8_t *tx_iv, + const uint8_t *current_rx_secret, const uint8_t *current_tx_secret, + size_t secretlen, void *user_data); /** * @functypedef @@ -1608,6 +1683,26 @@ typedef int (*ngtcp2_connection_id_status)(ngtcp2_conn *conn, int type, typedef int (*ngtcp2_recv_new_token)(ngtcp2_conn *conn, const ngtcp2_vec *token, void *user_data); +/** + * @functypedef + * + * :type:`ngtcp2_delete_crypto_aead_ctx` is a callback function which + * must delete the native object pointed by |aead_ctx|->native_handle. + */ +typedef void (*ngtcp2_delete_crypto_aead_ctx)(ngtcp2_conn *conn, + ngtcp2_crypto_aead_ctx *aead_ctx, + void *user_data); + +/** + * @functypedef + * + * :type:`ngtcp2_delete_crypto_cipher_ctx` is a callback function + * which must delete the native object pointed by + * |cipher_ctx|->native_handle. + */ +typedef void (*ngtcp2_delete_crypto_cipher_ctx)( + ngtcp2_conn *conn, ngtcp2_crypto_cipher_ctx *cipher_ctx, void *user_data); + typedef struct ngtcp2_conn_callbacks { /** * client_initial is a callback function which is invoked when @@ -1794,6 +1889,16 @@ typedef struct ngtcp2_conn_callbacks { * token is received from server. This field is ignored by server. */ ngtcp2_recv_new_token recv_new_token; + /** + * delete_crypto_aead_ctx is a callback function which deletes a + * given AEAD cipher context object. + */ + ngtcp2_delete_crypto_aead_ctx delete_crypto_aead_ctx; + /** + * delete_crypto_cipher_ctx is a callback function which deletes a + * given cipher context object. + */ + ngtcp2_delete_crypto_cipher_ctx delete_crypto_cipher_ctx; } ngtcp2_conn_callbacks; /** @@ -1820,9 +1925,9 @@ typedef struct ngtcp2_conn_callbacks { NGTCP2_EXTERN ngtcp2_ssize ngtcp2_pkt_write_connection_close( uint8_t *dest, size_t destlen, const ngtcp2_cid *dcid, const ngtcp2_cid *scid, uint64_t error_code, ngtcp2_encrypt encrypt, - const ngtcp2_crypto_aead *aead, const uint8_t *key, const uint8_t *iv, - ngtcp2_hp_mask hp_mask, const ngtcp2_crypto_cipher *hp, - const uint8_t *hp_key); + const ngtcp2_crypto_aead *aead, const ngtcp2_crypto_aead_ctx *aead_ctx, + const uint8_t *iv, ngtcp2_hp_mask hp_mask, const ngtcp2_crypto_cipher *hp, + const ngtcp2_crypto_cipher_ctx *hp_ctx); /** * @function @@ -1831,6 +1936,8 @@ NGTCP2_EXTERN ngtcp2_ssize ngtcp2_pkt_write_connection_close( * by |dest| whose length is |destlen|. |odcid| specifies Original * Destination Connection ID. |token| specifies Retry Token, and * |tokenlen| specifies its length. |aead| must be AEAD_AES_128_GCM. + * |aead_ctx| must be initialized with :macro:`NGTCP2_RETRY_KEY` as an + * encryption key. * * This function returns the number of bytes written to the buffer, or * one of the following negative error codes: @@ -1843,7 +1950,8 @@ NGTCP2_EXTERN ngtcp2_ssize ngtcp2_pkt_write_connection_close( NGTCP2_EXTERN ngtcp2_ssize ngtcp2_pkt_write_retry( uint8_t *dest, size_t destlen, const ngtcp2_cid *dcid, const ngtcp2_cid *scid, const ngtcp2_cid *odcid, const uint8_t *token, - size_t tokenlen, ngtcp2_encrypt encrypt, const ngtcp2_crypto_aead *aead); + size_t tokenlen, ngtcp2_encrypt encrypt, const ngtcp2_crypto_aead *aead, + const ngtcp2_crypto_aead_ctx *aead_ctx); /** * @function @@ -1983,12 +2091,21 @@ NGTCP2_EXTERN int ngtcp2_conn_get_handshake_completed(ngtcp2_conn *conn); * @function * * `ngtcp2_conn_install_initial_key` installs packet protection keying - * materials for Initial packets. |rx_key| of length |keylen|, IV - * |rx_iv| of length |rx_ivlen|, and packet header protection key - * |rx_hp_key| of length |keylen| to decrypt incoming Initial packets. - * Similarly, |tx_key|, |tx_iv| and |tx_hp_key| are for encrypt - * outgoing packets and are the same length with the rx counterpart . - * If they have already been set, they are overwritten. + * materials for Initial packets. |rx_aead_ctx| is AEAD cipher + * context object and must be initialized with decryption key, IV + * |rx_iv| of length |rx_ivlen|, and packet header protection cipher + * context object |rx_hp_ctx| to decrypt incoming Initial packets. + * Similarly, |tx_aead_ctx|, |tx_iv| and |tx_hp_ctx| are for + * encrypting outgoing packets and are the same length with the + * decryption counterpart . If they have already been set, they are + * overwritten. + * + * If this function succeeds, |conn| takes ownership of |rx_aead_ctx|, + * |rx_hp_ctx|, |tx_aead_ctx|, and |tx_hp_ctx|. + * :type:`ngtcp2_delete_crypto_aead_ctx` and + * :type:`ngtcp2_delete_crypto_cipher_ctx` will be called to delete + * these objects when they are no longer used. If this function + * fails, the caller is responsible to delete them. * * After receiving Retry packet, the DCID most likely changes. In * that case, client application must generate these keying materials @@ -2001,49 +2118,62 @@ NGTCP2_EXTERN int ngtcp2_conn_get_handshake_completed(ngtcp2_conn *conn); * Out of memory. */ NGTCP2_EXTERN int ngtcp2_conn_install_initial_key( - ngtcp2_conn *conn, const uint8_t *rx_key, const uint8_t *rx_iv, - const uint8_t *rx_hp_key, const uint8_t *tx_key, const uint8_t *tx_iv, - const uint8_t *tx_hp_key, size_t keylen, size_t ivlen); + ngtcp2_conn *conn, const ngtcp2_crypto_aead_ctx *rx_aead_ctx, + const uint8_t *rx_iv, const ngtcp2_crypto_cipher_ctx *rx_hp_ctx, + const ngtcp2_crypto_aead_ctx *tx_aead_ctx, const uint8_t *tx_iv, + const ngtcp2_crypto_cipher_ctx *tx_hp_ctx, size_t ivlen); /** * @function * * `ngtcp2_conn_install_rx_handshake_key` installs packet protection - * keying materials for decrypting incoming Handshake packets. |key| - * of length |keylen|, IV |iv| of length |ivlen|, and packet header - * protection key |hp_key| of length |keylen| to decrypt incoming + * keying materials for decrypting incoming Handshake packets. + * |aead_ctx| is AEAD cipher context object which must be initialized + * with decryption key, IV |iv| of length |ivlen|, and packet header + * protection cipher context object |hp_ctx| to decrypt incoming * Handshake packets. * + * If this function succeeds, |conn| takes ownership of |aead_ctx|, + * and |hp_ctx|. :type:`ngtcp2_delete_crypto_aead_ctx` and + * :type:`ngtcp2_delete_crypto_cipher_ctx` will be called to delete + * these objects when they are no longer used. If this function + * fails, the caller is responsible to delete them. + * * This function returns 0 if it succeeds, or one of the following * negative error codes: * * :enum:`NGTCP2_ERR_NOMEM` * Out of memory. */ -NGTCP2_EXTERN int -ngtcp2_conn_install_rx_handshake_key(ngtcp2_conn *conn, const uint8_t *key, - const uint8_t *iv, const uint8_t *hp_key, - size_t keylen, size_t ivlen); +NGTCP2_EXTERN int ngtcp2_conn_install_rx_handshake_key( + ngtcp2_conn *conn, const ngtcp2_crypto_aead_ctx *aead_ctx, + const uint8_t *iv, size_t ivlen, const ngtcp2_crypto_cipher_ctx *hp_ctx); /** * @function * * `ngtcp2_conn_install_tx_handshake_key` installs packet protection - * keying materials for encrypting outgoing Handshake packets. |key| - * of length |keylen|, IV |iv| of length |ivlen|, and packet header - * protection key |hp_key| of length |keylen| to encrypt outgoing + * keying materials for encrypting outgoing Handshake packets. + * |aead_ctx| is AEAD cipher context object which must be initialized + * with encryption key, IV |iv| of length |ivlen|, and packet header + * protection cipher context object |hp_ctx| to encrypt outgoing * Handshake packets. * + * If this function succeeds, |conn| takes ownership of |aead_ctx| and + * |hp_ctx|. :type:`ngtcp2_delete_crypto_aead_ctx` and + * :type:`ngtcp2_delete_crypto_cipher_ctx` will be called to delete + * these objects when they are no longer used. If this function + * fails, the caller is responsible to delete them. + * * This function returns 0 if it succeeds, or one of the following * negative error codes: * * :enum:`NGTCP2_ERR_NOMEM` * Out of memory. */ -NGTCP2_EXTERN int -ngtcp2_conn_install_tx_handshake_key(ngtcp2_conn *conn, const uint8_t *key, - const uint8_t *iv, const uint8_t *hp_key, - size_t keylen, size_t ivlen); +NGTCP2_EXTERN int ngtcp2_conn_install_tx_handshake_key( + ngtcp2_conn *conn, const ngtcp2_crypto_aead_ctx *aead_ctx, + const uint8_t *iv, size_t ivlen, const ngtcp2_crypto_cipher_ctx *hp_ctx); /** * @function @@ -2068,10 +2198,16 @@ NGTCP2_EXTERN size_t ngtcp2_conn_get_aead_overhead(ngtcp2_conn *conn); /** * @function * - * `ngtcp2_conn_install_early_key` installs packet protection key - * |key| of length |keylen|, IV |iv| of length |ivlen|, and packet - * header protection key |hp_key| of length |keylen| to encrypt (for - * client)or decrypt (for server) 0RTT packets. + * `ngtcp2_conn_install_early_key` installs packet protection AEAD + * cipher context object |aead_ctx|, IV |iv| of length |ivlen|, and + * packet header protection cipher context object |hp_ctx| to encrypt + * (for client) or decrypt (for server) 0RTT packets. + * + * If this function succeeds, |conn| takes ownership of |aead_ctx| and + * |hp_ctx|. :type:`ngtcp2_delete_crypto_aead_ctx` and + * :type:`ngtcp2_delete_crypto_cipher_ctx` will be called to delete + * these objects when they are no longer used. If this function + * fails, the caller is responsible to delete them. * * This function returns 0 if it succeeds, or one of the following * negative error codes: @@ -2079,11 +2215,9 @@ NGTCP2_EXTERN size_t ngtcp2_conn_get_aead_overhead(ngtcp2_conn *conn); * :enum:`NGTCP2_ERR_NOMEM` * Out of memory. */ -NGTCP2_EXTERN int ngtcp2_conn_install_early_key(ngtcp2_conn *conn, - const uint8_t *key, - const uint8_t *iv, - const uint8_t *hp_key, - size_t keylen, size_t ivlen); +NGTCP2_EXTERN int ngtcp2_conn_install_early_key( + ngtcp2_conn *conn, const ngtcp2_crypto_aead_ctx *aead_ctx, + const uint8_t *iv, size_t ivlen, const ngtcp2_crypto_cipher_ctx *hp_ctx); /** * @function @@ -2091,9 +2225,16 @@ NGTCP2_EXTERN int ngtcp2_conn_install_early_key(ngtcp2_conn *conn, * `ngtcp2_conn_install_rx_key` installs packet protection keying * materials for decrypting Short packets. |secret| of length * |secretlen| is the decryption secret which is used to derive keying - * materials passed to this function. |key| of length |keylen|, IV - * |iv| of length |ivlen|, and packet header protection key |hp_key| - * of length |keylen| to decrypt incoming Short packets. + * materials passed to this function. |aead_ctx| is AEAD cipher + * context object which must be initialized with decryption key, IV + * |iv| of length |ivlen|, and packet header protection cipher context + * object |hp_ctx| to decrypt incoming Short packets. + * + * If this function succeeds, |conn| takes ownership of |aead_ctx| and + * |hp_ctx|. :type:`ngtcp2_delete_crypto_aead_ctx` and + * :type:`ngtcp2_delete_crypto_cipher_ctx` will be called to delete + * these objects when they are no longer used. If this function + * fails, the caller is responsible to delete them. * * This function returns 0 if it succeeds, or one of the following * negative error codes: @@ -2101,11 +2242,10 @@ NGTCP2_EXTERN int ngtcp2_conn_install_early_key(ngtcp2_conn *conn, * :enum:`NGTCP2_ERR_NOMEM` * Out of memory. */ -NGTCP2_EXTERN int -ngtcp2_conn_install_rx_key(ngtcp2_conn *conn, const uint8_t *secret, - const uint8_t *key, const uint8_t *iv, - const uint8_t *hp_key, size_t secretlen, - size_t keylen, size_t ivlen); +NGTCP2_EXTERN int ngtcp2_conn_install_rx_key( + ngtcp2_conn *conn, const uint8_t *secret, size_t secretlen, + const ngtcp2_crypto_aead_ctx *aead_ctx, const uint8_t *iv, size_t ivlen, + const ngtcp2_crypto_cipher_ctx *hp_ctx); /** * @function @@ -2113,9 +2253,16 @@ ngtcp2_conn_install_rx_key(ngtcp2_conn *conn, const uint8_t *secret, * `ngtcp2_conn_install_tx_key` installs packet protection keying * materials for encrypting Short packets. |secret| of length * |secretlen| is the encryption secret which is used to derive keying - * materials passed to this function. |key| of length |keylen|, IV - * |iv| of length |ivlen|, and packet header protection key |hp_key| - * of length |keylen| to encrypt outgoing Short packets. + * materials passed to this function. |aead_ctx| is AEAD cipher + * context object which must be initialized with encryption key, IV + * |iv| of length |ivlen|, and packet header protection cipher context + * object |hp_ctx| to encrypt outgoing Short packets. + * + * If this function succeeds, |conn| takes ownership of |aead_ctx| and + * |hp_ctx|. :type:`ngtcp2_delete_crypto_aead_ctx` and + * :type:`ngtcp2_delete_crypto_cipher_ctx` will be called to delete + * these objects when they are no longer used. If this function + * fails, the caller is responsible to delete them. * * This function returns 0 if it succeeds, or one of the following * negative error codes: @@ -2123,11 +2270,10 @@ ngtcp2_conn_install_rx_key(ngtcp2_conn *conn, const uint8_t *secret, * :enum:`NGTCP2_ERR_NOMEM` * Out of memory. */ -NGTCP2_EXTERN int -ngtcp2_conn_install_tx_key(ngtcp2_conn *conn, const uint8_t *secret, - const uint8_t *key, const uint8_t *iv, - const uint8_t *hp_key, size_t secretlen, - size_t keylen, size_t ivlen); +NGTCP2_EXTERN int ngtcp2_conn_install_tx_key( + ngtcp2_conn *conn, const uint8_t *secret, size_t secretlen, + const ngtcp2_crypto_aead_ctx *aead_ctx, const uint8_t *iv, size_t ivlen, + const ngtcp2_crypto_cipher_ctx *hp_ctx); /** * @function @@ -2285,6 +2431,27 @@ ngtcp2_conn_get_remote_transport_params(ngtcp2_conn *conn, NGTCP2_EXTERN void ngtcp2_conn_set_early_remote_transport_params( ngtcp2_conn *conn, const ngtcp2_transport_params *params); +/** + * @function + * + * `ngtcp2_conn_set_local_transport_params` sets the local transport + * parameters |params|. This function can only be called by server. + * Although the local transport parameters are passed to + * `ngtcp2_conn_server_new`, server might want to update them after + * ALPN is chosen. In that case, server can update the transport + * parameter with this function. Server must call this function + * before calling `ngtcp2_conn_install_tx_handshake_key`. + * + * This function returns 0 if it succeeds, or one of the following + * negative error codes: + * + * :enum:`NGTCP2_ERR_INVALID_STATE` + * `ngtcp2_conn_install_tx_handshake_key` has been called. + */ +NGTCP2_EXTERN int +ngtcp2_conn_set_local_transport_params(ngtcp2_conn *conn, + const ngtcp2_transport_params *params); + /** * @function * @@ -2483,8 +2650,8 @@ NGTCP2_EXTERN ngtcp2_ssize ngtcp2_conn_write_stream( * packet is nearly full and the library decided to make a complete * packet. In this case, |*pdatalen| == -1 is asserted. * - * - The function returns :enum:`NGTCP2_ERR_WRITE_STREAM_MORE`. In - * this case, |*pdatalen| >= 0 is asserted. This indicates that + * - The function returns :enum:`NGTCP2_ERR_WRITE_MORE`. In this + * case, |*pdatalen| >= 0 is asserted. This indicates that * application can call this function with different stream data to * pack them into the same packet. Application has to specify the * same |conn|, |path|, |dest|, |destlen|, |pdatalen|, and |ts| @@ -2497,14 +2664,14 @@ NGTCP2_EXTERN ngtcp2_ssize ngtcp2_conn_write_stream( * - The other error might be returned just like without * :enum:`NGTCP2_WRITE_STREAM_FLAG_MORE`. * - * When application sees :enum:`NGTCP2_ERR_WRITE_STREAM_MORE`, it must - * not call other ngtcp2 API functions (application can still call + * When application sees :enum:`NGTCP2_ERR_WRITE_MORE`, it must not + * call other ngtcp2 API functions (application can still call * `ngtcp2_conn_write_connection_close` or * `ngtcp2_conn_write_application_close` to handle error from this * function). Just keep calling `ngtcp2_conn_writev_stream` or * `ngtcp2_conn_write_pkt` until it returns a positive number (which * indicates a complete packet is ready). If |*pdatalen| >= 0, the - * function always return :enum:`NGTCP2_ERR_WRITE_STREAM_MORE`. + * function always return :enum:`NGTCP2_ERR_WRITE_MORE`. * * This function returns 0 if it cannot write any frame because buffer * is too small, or packet is congestion limited. Application should @@ -2528,7 +2695,7 @@ NGTCP2_EXTERN ngtcp2_ssize ngtcp2_conn_write_stream( * User callback failed * :enum:`NGTCP2_ERR_STREAM_DATA_BLOCKED` * Stream is blocked because of flow control. - * :enum:`NGTCP2_ERR_WRITE_STREAM_MORE` + * :enum:`NGTCP2_ERR_WRITE_MORE` * (Only when :enum:`NGTCP2_WRITE_STREAM_FLAG_MORE` is specified) * Application can call this function to pack more stream data * into the same packet. See above to know how it works. @@ -2549,8 +2716,8 @@ NGTCP2_EXTERN ngtcp2_ssize ngtcp2_conn_writev_stream( * @function * * `ngtcp2_conn_write_connection_close` writes a packet which contains - * a CONNECTION_CLOSE frame in the buffer pointed by |dest| whose - * capacity is |datalen|. + * a CONNECTION_CLOSE frame (type 0x1c) in the buffer pointed by + * |dest| whose capacity is |datalen|. * * If |path| is not NULL, this function stores the network path with * which the packet should be sent. Each addr field must point to the @@ -2584,8 +2751,8 @@ NGTCP2_EXTERN ngtcp2_ssize ngtcp2_conn_write_connection_close( * @function * * `ngtcp2_conn_write_application_close` writes a packet which - * contains a APPLICATION_CLOSE frame in the buffer pointed by |dest| - * whose capacity is |datalen|. + * contains a CONNECTION_CLOSE frame (type 0x1d) in the buffer pointed + * by |dest| whose capacity is |datalen|. * * If |path| is not NULL, this function stores the network path with * which the packet should be sent. Each addr field must point to the @@ -2593,6 +2760,10 @@ NGTCP2_EXTERN ngtcp2_ssize ngtcp2_conn_write_connection_close( * sockaddr_storage)`` is enough. The assignment might not be done if * nothing is written to |dest|. * + * If handshake has not been confirmed yet, CONNECTION_CLOSE (type + * 0x1c) with error code :macro:`NGTCP2_APPLICATION_ERROR` is written + * instead. + * * This function must not be called from inside the callback * functions. * @@ -2605,7 +2776,7 @@ NGTCP2_EXTERN ngtcp2_ssize ngtcp2_conn_write_connection_close( * :enum:`NGTCP2_ERR_NOBUF` * Buffer is too small * :enum:`NGTCP2_ERR_INVALID_STATE` - * The current state does not allow sending APPLICATION_CLOSE. + * The current state does not allow sending CONNECTION_CLOSE. * :enum:`NGTCP2_ERR_PKT_NUM_EXHAUSTED` * Packet number is exhausted, and cannot send any more packet. * :enum:`NGTCP2_ERR_CALLBACK_FAILURE` @@ -2974,13 +3145,21 @@ NGTCP2_EXTERN void ngtcp2_conn_set_tls_native_handle(ngtcp2_conn *conn, /** * @function * - * `ngtcp2_conn_set_retry_aead` sets |aead| for Retry integrity tag - * verification. It must be AEAD_AES_128_GCM. This function must be - * called if |conn| is initialized as client. Server does not verify - * the tag and has no need to call this function. + * `ngtcp2_conn_set_retry_aead` sets |aead| and |aead_ctx| for Retry + * integrity tag verification. |aead| must be AEAD_AES_128_GCM. + * |aead_ctx| must be initialized with :macro:`NGTCP2_RETRY_KEY` as + * encryption key. This function must be called if |conn| is + * initialized as client. Server does not verify the tag and has no + * need to call this function. + * + * If this function succeeds, |conn| takes ownership of |aead_ctx|. + * :type:`ngtcp2_delete_crypto_aead_ctx` will be called to delete this + * object when it is no longer used. If this function fails, the + * caller is responsible to delete it. */ -NGTCP2_EXTERN void ngtcp2_conn_set_retry_aead(ngtcp2_conn *conn, - const ngtcp2_crypto_aead *aead); +NGTCP2_EXTERN void +ngtcp2_conn_set_retry_aead(ngtcp2_conn *conn, const ngtcp2_crypto_aead *aead, + const ngtcp2_crypto_aead_ctx *aead_ctx); /** * @function @@ -3033,6 +3212,30 @@ NGTCP2_EXTERN int ngtcp2_conn_is_local_stream(ngtcp2_conn *conn, */ NGTCP2_EXTERN int ngtcp2_conn_is_server(ngtcp2_conn *conn); +/** + * @function + * + * `ngtcp2_conn_after_retry` returns nonzero if |conn| as a client has + * received Retry packet from server and successfully validated it. + */ +NGTCP2_EXTERN int ngtcp2_conn_after_retry(ngtcp2_conn *conn); + +/** + * @function + * + * `ngtcp2_conn_set_stream_user_data` sets |stream_user_data| to the + * stream identified by |stream_id|. + * + * This function returns 0 if it succeeds, or one of the following + * negative error codes: + * + * :enum:`NGTCP2_ERR_STREAM_NOT_FOUND` + * Stream does not exist + */ +NGTCP2_EXTERN int ngtcp2_conn_set_stream_user_data(ngtcp2_conn *conn, + int64_t stream_id, + void *stream_user_data); + /** * @function * @@ -3061,7 +3264,8 @@ NGTCP2_EXTERN uint64_t ngtcp2_err_infer_quic_transport_error_code(int liberr); * `ngtcp2_addr_init` initializes |dest| with the given arguments and * returns |dest|. */ -NGTCP2_EXTERN ngtcp2_addr *ngtcp2_addr_init(ngtcp2_addr *dest, const void *addr, +NGTCP2_EXTERN ngtcp2_addr *ngtcp2_addr_init(ngtcp2_addr *dest, + const struct sockaddr *addr, size_t addrlen, void *user_data); /** @@ -3070,11 +3274,13 @@ NGTCP2_EXTERN ngtcp2_addr *ngtcp2_addr_init(ngtcp2_addr *dest, const void *addr, * `ngtcp2_path_storage_init` initializes |ps| with the given * arguments. This function copies |local_addr| and |remote_addr|. */ -NGTCP2_EXTERN void -ngtcp2_path_storage_init(ngtcp2_path_storage *ps, const void *local_addr, - size_t local_addrlen, void *local_user_data, - const void *remote_addr, size_t remote_addrlen, - void *remote_user_data); +NGTCP2_EXTERN void ngtcp2_path_storage_init(ngtcp2_path_storage *ps, + const struct sockaddr *local_addr, + size_t local_addrlen, + void *local_user_data, + const struct sockaddr *remote_addr, + size_t remote_addrlen, + void *remote_user_data); /** * @function @@ -3092,6 +3298,7 @@ NGTCP2_EXTERN void ngtcp2_path_storage_zero(ngtcp2_path_storage *ps); * default value to the following fields: * * * cc_algo = NGTCP2_CC_ALGO_CUBIC + * * initial_rtt = NGTCP2_DEFAULT_INITIAL_RTT * * transport_params.max_udp_payload_size = NGTCP2_DEFAULT_MAX_UDP_PAYLOAD_SIZE * * transport_params.ack_delay_component = NGTCP2_DEFAULT_ACK_DELAY_EXPONENT * * transport_params.max_ack_delay = NGTCP2_DEFAULT_MAX_ACK_DELAY diff --git a/deps/ngtcp2/lib/ngtcp2_acktr.c b/deps/ngtcp2/lib/ngtcp2_acktr.c index 85113716b1c872..7a7f3e469a2f0e 100644 --- a/deps/ngtcp2/lib/ngtcp2_acktr.c +++ b/deps/ngtcp2/lib/ngtcp2_acktr.c @@ -297,8 +297,6 @@ void ngtcp2_acktr_recv_ack(ngtcp2_acktr *acktr, const ngtcp2_ack *fr) { return; } } - - return; } void ngtcp2_acktr_commit_ack(ngtcp2_acktr *acktr) { diff --git a/deps/ngtcp2/lib/ngtcp2_addr.c b/deps/ngtcp2/lib/ngtcp2_addr.c index 13a787bdea0302..cfc91c41e2c38e 100644 --- a/deps/ngtcp2/lib/ngtcp2_addr.c +++ b/deps/ngtcp2/lib/ngtcp2_addr.c @@ -25,11 +25,24 @@ #include "ngtcp2_addr.h" #include +#include -ngtcp2_addr *ngtcp2_addr_init(ngtcp2_addr *dest, const void *addr, +#ifdef WIN32 +# include +# include +#else +# include +# include +# include +# include +# include +# include +#endif + +ngtcp2_addr *ngtcp2_addr_init(ngtcp2_addr *dest, const struct sockaddr *addr, size_t addrlen, void *user_data) { dest->addrlen = addrlen; - dest->addr = (uint8_t *)addr; + dest->addr = (struct sockaddr *)addr; dest->user_data = user_data; return dest; } @@ -42,7 +55,7 @@ void ngtcp2_addr_copy(ngtcp2_addr *dest, const ngtcp2_addr *src) { dest->user_data = src->user_data; } -void ngtcp2_addr_copy_byte(ngtcp2_addr *dest, const void *addr, +void ngtcp2_addr_copy_byte(ngtcp2_addr *dest, const struct sockaddr *addr, size_t addrlen) { dest->addrlen = addrlen; if (addrlen) { @@ -50,8 +63,30 @@ void ngtcp2_addr_copy_byte(ngtcp2_addr *dest, const void *addr, } } +static int sockaddr_eq(const struct sockaddr *a, const struct sockaddr *b) { + assert(a->sa_family == b->sa_family); + + switch (a->sa_family) { + case AF_INET: { + const struct sockaddr_in *ai = (const struct sockaddr_in *)(void *)a, + *bi = (const struct sockaddr_in *)(void *)b; + return ai->sin_port == bi->sin_port && + memcmp(&ai->sin_addr, &bi->sin_addr, sizeof(ai->sin_addr)) == 0; + } + case AF_INET6: { + const struct sockaddr_in6 *ai = (const struct sockaddr_in6 *)(void *)a, + *bi = (const struct sockaddr_in6 *)(void *)b; + return ai->sin6_port == bi->sin6_port && + memcmp(&ai->sin6_addr, &bi->sin6_addr, sizeof(ai->sin6_addr)) == 0; + } + default: + assert(0); + } +} + int ngtcp2_addr_eq(const ngtcp2_addr *a, const ngtcp2_addr *b) { - return a->addrlen == b->addrlen && memcmp(a->addr, b->addr, a->addrlen) == 0; + return a->addr->sa_family == b->addr->sa_family && + sockaddr_eq(a->addr, b->addr); } int ngtcp2_addr_empty(const ngtcp2_addr *addr) { return addr->addrlen == 0; } diff --git a/deps/ngtcp2/lib/ngtcp2_addr.h b/deps/ngtcp2/lib/ngtcp2_addr.h index db8b7144082a14..238bb435183c20 100644 --- a/deps/ngtcp2/lib/ngtcp2_addr.h +++ b/deps/ngtcp2/lib/ngtcp2_addr.h @@ -44,7 +44,8 @@ void ngtcp2_addr_copy(ngtcp2_addr *dest, const ngtcp2_addr *src); * |addrlen|. This function assumes that dest->addr points to a * buffer which have sufficient size to store the copy. */ -void ngtcp2_addr_copy_byte(ngtcp2_addr *dest, const void *addr, size_t addrlen); +void ngtcp2_addr_copy_byte(ngtcp2_addr *dest, const struct sockaddr *addr, + size_t addrlen); /* * ngtcp2_addr_eq returns nonzero if |a| equals |b|. diff --git a/deps/ngtcp2/lib/ngtcp2_cc.c b/deps/ngtcp2/lib/ngtcp2_cc.c index 3aed4dd759dd90..ef2e63a0efdac0 100644 --- a/deps/ngtcp2/lib/ngtcp2_cc.c +++ b/deps/ngtcp2/lib/ngtcp2_cc.c @@ -26,24 +26,15 @@ #include +#if defined(_MSC_VER) +# include +#endif + #include "ngtcp2_log.h" #include "ngtcp2_macro.h" #include "ngtcp2_mem.h" #include "ngtcp2_rcvry.h" -#ifdef _MSC_VER -#include -static inline int __builtin_clzll(unsigned long long x) { -#if defined(_WIN64) || defined(_LP64) - return (int)__lzcnt64(x); -#else - // TODO(@jasnell): Determine if there's an alternative available for x86 - assert(0); -#endif - -} -#endif - uint64_t ngtcp2_cc_compute_initcwnd(size_t max_udp_payload_size) { uint64_t n = 2 * max_udp_payload_size; n = ngtcp2_max(n, 14720); @@ -245,16 +236,35 @@ void ngtcp2_cc_cubic_cc_free(ngtcp2_cc *cc, const ngtcp2_mem *mem) { static uint64_t ngtcp2_cbrt(uint64_t n) { int d; uint64_t a; - int i; if (n == 0) { return 0; } +#if defined(_MSC_VER) +# if defined(_M_X64) + d = (int)__lzcnt64(n); +# elif defined(_M_ARM64) + { + unsigned long index; + d = sizeof(uint64_t) * CHAR_BIT; + if (_BitScanReverse64(&index, n)) { + d = d - 1 - index; + } + } +# else + if ((n >> 32) != 0) { + d = __lzcnt((unsigned int)(n >> 32)); + } else { + d = 32 + __lzcnt((unsigned int)n); + } +# endif +#else d = __builtin_clzll(n); +#endif a = 1ULL << ((64 - d) / 3 + 1); - for (i = 0; a * a * a > n; ++i) { + for (; a * a * a > n;) { a = (2 * a + n / a / a) / 3; } return a; @@ -274,6 +284,7 @@ void ngtcp2_cc_cubic_cc_on_pkt_acked(ngtcp2_cc *ccx, ngtcp2_conn_stat *cstat, uint64_t target; uint64_t tx, kx, time_delta, delta; uint64_t add, tcp_add; + uint64_t m; if (pkt->pktns_id == NGTCP2_PKTNS_ID_APP && cc->window_end != -1 && cc->window_end <= pkt->pkt_num) { @@ -339,10 +350,12 @@ void ngtcp2_cc_cubic_cc_on_pkt_acked(ngtcp2_cc *ccx, ngtcp2_conn_stat *cstat, "cubic-ca epoch_start=%" PRIu64 " k=%" PRIu64 " origin_point=%" PRIu64, cc->epoch_start, cc->k, cc->origin_point); + + cc->pending_add = 0; + cc->pending_w_add = 0; } - min_rtt = cstat->min_rtt == UINT64_MAX ? NGTCP2_DEFAULT_INITIAL_RTT - : cstat->min_rtt; + min_rtt = cstat->min_rtt == UINT64_MAX ? cstat->initial_rtt : cstat->min_rtt; t = ts + min_rtt - cc->epoch_start; @@ -365,13 +378,19 @@ void ngtcp2_cc_cubic_cc_on_pkt_acked(ngtcp2_cc *ccx, ngtcp2_conn_stat *cstat, } if (target > cstat->cwnd) { - add = cstat->max_udp_payload_size * (target - cstat->cwnd) / cstat->cwnd; + m = cc->pending_add + cstat->max_udp_payload_size * (target - cstat->cwnd); + add = m / cstat->cwnd; + cc->pending_add = m % cstat->cwnd; } else { - /* TODO too small, no increment at all */ - add = cstat->max_udp_payload_size / (100 * cstat->cwnd); + m = cc->pending_add + cstat->max_udp_payload_size; + add = m / (100 * cstat->cwnd); + cc->pending_add = m % (100 * cstat->cwnd); } - cc->w_tcp += cstat->max_udp_payload_size * pkt->pktlen * 9 / 17 / cstat->cwnd; + m = cc->pending_w_add + cstat->max_udp_payload_size * pkt->pktlen; + + cc->w_tcp += m / cstat->cwnd; + cc->pending_w_add = m % cstat->cwnd; if (cc->w_tcp > cstat->cwnd) { tcp_add = diff --git a/deps/ngtcp2/lib/ngtcp2_cc.h b/deps/ngtcp2/lib/ngtcp2_cc.h index c3886a6e6b41cb..05010d57251c39 100644 --- a/deps/ngtcp2/lib/ngtcp2_cc.h +++ b/deps/ngtcp2/lib/ngtcp2_cc.h @@ -95,6 +95,8 @@ typedef struct ngtcp2_cubic_cc { uint64_t current_round_min_rtt; uint64_t last_round_min_rtt; int64_t window_end; + uint64_t pending_add; + uint64_t pending_w_add; } ngtcp2_cubic_cc; int ngtcp2_cc_cubic_cc_init(ngtcp2_cc *cc, ngtcp2_log *log, diff --git a/deps/ngtcp2/lib/ngtcp2_cid.c b/deps/ngtcp2/lib/ngtcp2_cid.c index b0670a9c6f0579..126e3c4a512e80 100644 --- a/deps/ngtcp2/lib/ngtcp2_cid.c +++ b/deps/ngtcp2/lib/ngtcp2_cid.c @@ -105,7 +105,6 @@ void ngtcp2_dcid_copy_no_path(ngtcp2_dcid *dest, const ngtcp2_dcid *src) { int ngtcp2_dcid_verify_uniqueness(ngtcp2_dcid *dcid, uint64_t seq, const ngtcp2_cid *cid, const uint8_t *token) { - if (dcid->seq == seq) { return ngtcp2_cid_eq(&dcid->cid, cid) && memcmp(dcid->token, token, diff --git a/deps/ngtcp2/lib/ngtcp2_conn.c b/deps/ngtcp2/lib/ngtcp2_conn.c index 41e5f4d915e3e4..ff8c608b776dac 100644 --- a/deps/ngtcp2/lib/ngtcp2_conn.c +++ b/deps/ngtcp2/lib/ngtcp2_conn.c @@ -359,6 +359,29 @@ static int conn_call_deactivate_dcid(ngtcp2_conn *conn, conn, NGTCP2_CONNECTION_ID_STATUS_TYPE_DEACTIVATE, dcid); } +static void conn_call_delete_crypto_aead_ctx(ngtcp2_conn *conn, + ngtcp2_crypto_aead_ctx *aead_ctx) { + if (!aead_ctx->native_handle) { + return; + } + + assert(conn->callbacks.delete_crypto_aead_ctx); + + conn->callbacks.delete_crypto_aead_ctx(conn, aead_ctx, conn->user_data); +} + +static void +conn_call_delete_crypto_cipher_ctx(ngtcp2_conn *conn, + ngtcp2_crypto_cipher_ctx *cipher_ctx) { + if (!cipher_ctx->native_handle) { + return; + } + + assert(conn->callbacks.delete_crypto_cipher_ctx); + + conn->callbacks.delete_crypto_cipher_ctx(conn, cipher_ctx, conn->user_data); +} + static int crypto_offset_less(const ngtcp2_ksl_key *lhs, const ngtcp2_ksl_key *rhs) { return *(int64_t *)lhs < *(int64_t *)rhs; @@ -458,9 +481,6 @@ static void pktns_free(ngtcp2_pktns *pktns, const ngtcp2_mem *mem) { ngtcp2_frame_chain_list_del(pktns->tx.frq, mem); - ngtcp2_vec_del(pktns->crypto.rx.hp_key, mem); - ngtcp2_vec_del(pktns->crypto.tx.hp_key, mem); - ngtcp2_crypto_km_del(pktns->crypto.rx.ckm, mem); ngtcp2_crypto_km_del(pktns->crypto.tx.ckm, mem); @@ -513,25 +533,46 @@ static int ts_retired_less(const ngtcp2_pq_entry *lhs, return a->ts_retired < b->ts_retired; } -static void conn_reset_conn_stat(ngtcp2_conn *conn, ngtcp2_conn_stat *cstat) { +/* + * conn_reset_conn_stat_cc resets congestion state in |cstat|. + */ +static void conn_reset_conn_stat_cc(ngtcp2_conn *conn, + ngtcp2_conn_stat *cstat) { cstat->latest_rtt = 0; cstat->min_rtt = UINT64_MAX; - cstat->smoothed_rtt = NGTCP2_DEFAULT_INITIAL_RTT; - cstat->rttvar = NGTCP2_DEFAULT_INITIAL_RTT / 2; + cstat->smoothed_rtt = conn->local.settings.initial_rtt; + cstat->rttvar = conn->local.settings.initial_rtt / 2; cstat->pto_count = 0; cstat->loss_detection_timer = 0; - // Initializes them with UINT64_MAX. - memset(cstat->loss_time, 0xff, sizeof(cstat->loss_time)); cstat->cwnd = ngtcp2_cc_compute_initcwnd(conn->local.settings.max_udp_payload_size); cstat->ssthresh = UINT64_MAX; cstat->congestion_recovery_start_ts = 0; cstat->bytes_in_flight = 0; - cstat->max_udp_payload_size = conn->local.settings.max_udp_payload_size; cstat->delivery_rate_sec = 0; cstat->recv_rate_sec = 0; } +/* + * reset_conn_stat_recovery resets the fields related to the recovery + * function + */ +static void reset_conn_stat_recovery(ngtcp2_conn_stat *cstat) { + // Initializes them with UINT64_MAX. + memset(cstat->loss_time, 0xff, sizeof(cstat->loss_time)); + memset(cstat->last_tx_pkt_ts, 0xff, sizeof(cstat->last_tx_pkt_ts)); +} + +/* + * conn_reset_conn_stat resets |cstat|. The following fields are not + * reset: initial_rtt, max_udp_payload_size, bytes_sent, and + * bytes_recv. + */ +static void conn_reset_conn_stat(ngtcp2_conn *conn, ngtcp2_conn_stat *cstat) { + conn_reset_conn_stat_cc(conn, cstat); + reset_conn_stat_recovery(cstat); +} + static void conn_reset_rx_rate(ngtcp2_conn *conn) { conn->rx.rate.start_ts = UINT64_MAX; conn->rx.rate.received = 0; @@ -551,9 +592,15 @@ static void conn_update_recv_rate(ngtcp2_conn *conn, size_t datalen, assert(conn->cstat.min_rtt); - window = conn->cstat.min_rtt == UINT64_MAX ? NGTCP2_DEFAULT_INITIAL_RTT + window = conn->cstat.min_rtt == UINT64_MAX ? conn->cstat.initial_rtt : conn->cstat.min_rtt * 2; + /* If settings.initial_rtt is zero for whatever reason then window + can be zero and we can end up with a division by zero error when + bps is set below. If this assert fails, check that + settings.initial_rtt is not zero. */ + assert(window); + if (window > ts - conn->rx.rate.start_ts) { return; } @@ -621,6 +668,11 @@ static int conn_new(ngtcp2_conn **pconn, const ngtcp2_cid *dcid, goto fail_dcid_retired_init; } + rv = ngtcp2_gaptr_init(&(*pconn)->dcid.seqgap, mem); + if (rv != 0) { + goto fail_seqgap_init; + } + rv = ngtcp2_ksl_init(&(*pconn)->scid.set, cid_less, sizeof(ngtcp2_cid), mem); if (rv != 0) { goto fail_scid_set_init; @@ -677,22 +729,14 @@ static int conn_new(ngtcp2_conn **pconn, const ngtcp2_cid *dcid, (*pconn)->local.settings.token.len = 0; } - if (params->active_connection_id_limit == 0) { - (*pconn)->local.settings.transport_params.active_connection_id_limit = - NGTCP2_DEFAULT_ACTIVE_CONNECTION_ID_LIMIT; - } - - (*pconn)->local.settings.transport_params.initial_scid = *scid; - - if (scid->datalen == 0) { - (*pconn)->local.settings.transport_params.preferred_address_present = 0; - } - if (settings->max_udp_payload_size == 0) { (*pconn)->local.settings.max_udp_payload_size = NGTCP2_DEFAULT_MAX_PKTLEN; } conn_reset_conn_stat(*pconn, &(*pconn)->cstat); + (*pconn)->cstat.initial_rtt = settings->initial_rtt; + (*pconn)->cstat.max_udp_payload_size = + (*pconn)->local.settings.max_udp_payload_size; ngtcp2_rst_init(&(*pconn)->rst); @@ -746,10 +790,9 @@ static int conn_new(ngtcp2_conn **pconn, const ngtcp2_cid *dcid, goto fail_scident; } - ngtcp2_scid_init(scident, 0, scid, - params->stateless_reset_token_present - ? params->stateless_reset_token - : NULL); + /* Set stateless reset token later if it is available in the local + transport parameters */ + ngtcp2_scid_init(scident, 0, scid, NULL); rv = ngtcp2_ksl_insert(&(*pconn)->scid.set, NULL, &scident->cid, scident); if (rv != 0) { @@ -758,52 +801,29 @@ static int conn_new(ngtcp2_conn **pconn, const ngtcp2_cid *dcid, scident = NULL; - if (server && params->preferred_address_present) { - scident = ngtcp2_mem_malloc(mem, sizeof(*scident)); - if (scident == NULL) { - rv = NGTCP2_ERR_NOMEM; - goto fail_scident; - } - - ngtcp2_scid_init(scident, 1, ¶ms->preferred_address.cid, - params->preferred_address.stateless_reset_token); - - rv = ngtcp2_ksl_insert(&(*pconn)->scid.set, NULL, &scident->cid, scident); - if (rv != 0) { - goto fail_scid_set_insert; - } - - scident = NULL; - - (*pconn)->scid.last_seq = 1; - } - ngtcp2_dcid_init(&(*pconn)->dcid.current, 0, dcid, NULL); ngtcp2_path_copy(&(*pconn)->dcid.current.ps.path, path); + rv = ngtcp2_gaptr_push(&(*pconn)->dcid.seqgap, 0, 1); + if (rv != 0) { + goto fail_seqgap_push; + } + + (*pconn)->server = server; (*pconn)->oscid = *scid; (*pconn)->callbacks = *callbacks; (*pconn)->version = version; (*pconn)->mem = mem; (*pconn)->user_data = user_data; - (*pconn)->rx.unsent_max_offset = (*pconn)->rx.max_offset = - params->initial_max_data; - (*pconn)->remote.bidi.unsent_max_streams = params->initial_max_streams_bidi; - (*pconn)->remote.bidi.max_streams = params->initial_max_streams_bidi; - (*pconn)->remote.uni.unsent_max_streams = params->initial_max_streams_uni; - (*pconn)->remote.uni.max_streams = params->initial_max_streams_uni; (*pconn)->idle_ts = settings->initial_ts; (*pconn)->crypto.key_update.confirmed_ts = UINT64_MAX; ngtcp2_qlog_start(&(*pconn)->qlog, server ? &settings->qlog.odcid : dcid, server); - ngtcp2_qlog_parameters_set_transport_params( - &(*pconn)->qlog, &(*pconn)->local.settings.transport_params, - (*pconn)->server, - /* local = */ 1); return 0; +fail_seqgap_push: fail_scid_set_insert: ngtcp2_mem_free(mem, scident); fail_scident: @@ -830,6 +850,8 @@ static int conn_new(ngtcp2_conn **pconn, const ngtcp2_cid *dcid, delete_scid(&(*pconn)->scid.set, mem); ngtcp2_ksl_free(&(*pconn)->scid.set); fail_scid_set_init: + ngtcp2_gaptr_free(&(*pconn)->dcid.seqgap); +fail_seqgap_init: ngtcp2_ringbuf_free(&(*pconn)->dcid.retired); fail_dcid_retired_init: ngtcp2_ringbuf_free(&(*pconn)->dcid.unused); @@ -856,6 +878,16 @@ int ngtcp2_conn_client_new(ngtcp2_conn **pconn, const ngtcp2_cid *dcid, (*pconn)->local.bidi.next_stream_id = 0; (*pconn)->local.uni.next_stream_id = 2; + rv = ngtcp2_conn_commit_local_transport_params(*pconn); + if (rv != 0) { + ngtcp2_conn_del(*pconn); + return rv; + } + + ngtcp2_qlog_parameters_set_transport_params( + &(*pconn)->qlog, &(*pconn)->local.settings.transport_params, + (*pconn)->server, NGTCP2_QLOG_SIDE_LOCAL); + return 0; } @@ -866,24 +898,15 @@ int ngtcp2_conn_server_new(ngtcp2_conn **pconn, const ngtcp2_cid *dcid, const ngtcp2_settings *settings, const ngtcp2_mem *mem, void *user_data) { int rv; - ngtcp2_transport_params *params; - rv = conn_new(pconn, dcid, scid, path, version, callbacks, settings, mem, user_data, 1); if (rv != 0) { return rv; } - (*pconn)->server = 1; (*pconn)->state = NGTCP2_CS_SERVER_INITIAL; (*pconn)->local.bidi.next_stream_id = 1; (*pconn)->local.uni.next_stream_id = 3; - params = &(*pconn)->local.settings.transport_params; - if (dcid->datalen == 0) { - /* Client uses zero-length Connection ID */ - params->active_connection_id_limit = 0; - } - if ((*pconn)->local.settings.token.len) { /* Usage of token lifts amplification limit */ (*pconn)->flags |= NGTCP2_CONN_FLAG_SADDR_VERIFIED; @@ -930,13 +953,71 @@ void ngtcp2_conn_del(ngtcp2_conn *conn) { ngtcp2_qlog_end(&conn->qlog); + if (conn->early.ckm) { + conn_call_delete_crypto_aead_ctx(conn, &conn->early.ckm->aead_ctx); + } + conn_call_delete_crypto_cipher_ctx(conn, &conn->early.hp_ctx); + + if (conn->crypto.key_update.old_rx_ckm) { + conn_call_delete_crypto_aead_ctx( + conn, &conn->crypto.key_update.old_rx_ckm->aead_ctx); + } + if (conn->crypto.key_update.new_rx_ckm) { + conn_call_delete_crypto_aead_ctx( + conn, &conn->crypto.key_update.new_rx_ckm->aead_ctx); + } + if (conn->crypto.key_update.new_tx_ckm) { + conn_call_delete_crypto_aead_ctx( + conn, &conn->crypto.key_update.new_tx_ckm->aead_ctx); + } + + if (conn->pktns.crypto.rx.ckm) { + conn_call_delete_crypto_aead_ctx(conn, + &conn->pktns.crypto.rx.ckm->aead_ctx); + } + conn_call_delete_crypto_cipher_ctx(conn, &conn->pktns.crypto.rx.hp_ctx); + + if (conn->pktns.crypto.tx.ckm) { + conn_call_delete_crypto_aead_ctx(conn, + &conn->pktns.crypto.tx.ckm->aead_ctx); + } + conn_call_delete_crypto_cipher_ctx(conn, &conn->pktns.crypto.tx.hp_ctx); + + if (conn->hs_pktns) { + if (conn->hs_pktns->crypto.rx.ckm) { + conn_call_delete_crypto_aead_ctx( + conn, &conn->hs_pktns->crypto.rx.ckm->aead_ctx); + } + conn_call_delete_crypto_cipher_ctx(conn, &conn->hs_pktns->crypto.rx.hp_ctx); + + if (conn->hs_pktns->crypto.tx.ckm) { + conn_call_delete_crypto_aead_ctx( + conn, &conn->hs_pktns->crypto.tx.ckm->aead_ctx); + } + conn_call_delete_crypto_cipher_ctx(conn, &conn->hs_pktns->crypto.tx.hp_ctx); + } + if (conn->in_pktns) { + if (conn->in_pktns->crypto.rx.ckm) { + conn_call_delete_crypto_aead_ctx( + conn, &conn->in_pktns->crypto.rx.ckm->aead_ctx); + } + conn_call_delete_crypto_cipher_ctx(conn, &conn->in_pktns->crypto.rx.hp_ctx); + + if (conn->in_pktns->crypto.tx.ckm) { + conn_call_delete_crypto_aead_ctx( + conn, &conn->in_pktns->crypto.tx.ckm->aead_ctx); + } + conn_call_delete_crypto_cipher_ctx(conn, &conn->in_pktns->crypto.tx.hp_ctx); + } + + conn_call_delete_crypto_aead_ctx(conn, &conn->crypto.retry_aead_ctx); + ngtcp2_mem_free(conn->mem, conn->crypto.decrypt_buf.base); ngtcp2_mem_free(conn->mem, conn->local.settings.token.base); ngtcp2_crypto_km_del(conn->crypto.key_update.old_rx_ckm, conn->mem); ngtcp2_crypto_km_del(conn->crypto.key_update.new_rx_ckm, conn->mem); ngtcp2_crypto_km_del(conn->crypto.key_update.new_tx_ckm, conn->mem); - ngtcp2_vec_del(conn->early.hp_key, conn->mem); ngtcp2_crypto_km_del(conn->early.ckm, conn->mem); pktns_free(&conn->pktns, conn->mem); @@ -961,6 +1042,7 @@ void ngtcp2_conn_del(ngtcp2_conn *conn) { ngtcp2_pq_free(&conn->scid.used); delete_scid(&conn->scid.set, conn->mem); ngtcp2_ksl_free(&conn->scid.set); + ngtcp2_gaptr_free(&conn->dcid.seqgap); ngtcp2_ringbuf_free(&conn->dcid.retired); ngtcp2_ringbuf_free(&conn->dcid.unused); @@ -1031,18 +1113,20 @@ static ngtcp2_duration conn_compute_ack_delay(ngtcp2_conn *conn) { * Out of memory. */ static int conn_create_ack_frame(ngtcp2_conn *conn, ngtcp2_frame **pfr, - ngtcp2_acktr *acktr, uint8_t type, + ngtcp2_pktns *pktns, uint8_t type, ngtcp2_tstamp ts, ngtcp2_duration ack_delay, uint64_t ack_delay_exponent) { - /* TODO Measure an actual size of ACK bloks to find the best default - value. */ + /* TODO Measure an actual size of ACK blocks to find the best + default value. */ const size_t initial_max_ack_blks = 8; int64_t last_pkt_num; + ngtcp2_acktr *acktr = &pktns->acktr; ngtcp2_ack_blk *blk; ngtcp2_ksl_it it; ngtcp2_acktr_entry *rpkt; ngtcp2_ack *ack; size_t blk_idx; + ngtcp2_tstamp largest_ack_ts; int rv; if (acktr->flags & NGTCP2_ACKTR_FLAG_IMMEDIATE_ACK) { @@ -1071,22 +1155,35 @@ static int conn_create_ack_frame(ngtcp2_conn *conn, ngtcp2_frame **pfr, ack = &conn->tx.ack->ack; - rpkt = ngtcp2_ksl_it_get(&it); - last_pkt_num = rpkt->pkt_num - (int64_t)(rpkt->len - 1); ack->type = NGTCP2_FRAME_ACK; - ack->largest_ack = rpkt->pkt_num; - ack->first_ack_blklen = rpkt->len - 1; + ack->num_blks = 0; + + rpkt = ngtcp2_ksl_it_get(&it); + + if (rpkt->pkt_num == pktns->rx.max_pkt_num) { + last_pkt_num = rpkt->pkt_num - (int64_t)(rpkt->len - 1); + largest_ack_ts = rpkt->tstamp; + ack->largest_ack = rpkt->pkt_num; + ack->first_ack_blklen = rpkt->len - 1; + + ngtcp2_ksl_it_next(&it); + } else { + assert(rpkt->pkt_num < pktns->rx.max_pkt_num); + + last_pkt_num = pktns->rx.max_pkt_num; + largest_ack_ts = pktns->rx.max_pkt_ts; + ack->largest_ack = pktns->rx.max_pkt_num; + ack->first_ack_blklen = 0; + } + if (type == NGTCP2_PKT_SHORT) { - ack->ack_delay_unscaled = ts - rpkt->tstamp; + ack->ack_delay_unscaled = ts - largest_ack_ts; ack->ack_delay = ack->ack_delay_unscaled / NGTCP2_MICROSECONDS / (1UL << ack_delay_exponent); } else { ack->ack_delay_unscaled = 0; ack->ack_delay = 0; } - ack->num_blks = 0; - - ngtcp2_ksl_it_next(&it); for (; !ngtcp2_ksl_it_end(&it); ngtcp2_ksl_it_next(&it)) { if (ack->num_blks == NGTCP2_MAX_ACK_BLKS) { @@ -1189,11 +1286,10 @@ static int conn_on_pkt_sent(ngtcp2_conn *conn, ngtcp2_rtb *rtb, if (ent->flags & NGTCP2_RTB_FLAG_ACK_ELICITING) { conn->cstat.last_tx_pkt_ts[rtb->pktns_id] = ent->ts; - /* I think pseudo code is wrong; timer should be set when - ack-eliciting packet is sent. */ - ngtcp2_conn_set_loss_detection_timer(conn, ent->ts); } + ngtcp2_conn_set_loss_detection_timer(conn, ent->ts); + return 0; } @@ -1264,19 +1360,52 @@ static size_t conn_retry_early_payloadlen(ngtcp2_conn *conn) { return 0; } +static void conn_cryptofrq_clear(ngtcp2_conn *conn, ngtcp2_pktns *pktns) { + ngtcp2_frame_chain *frc; + ngtcp2_ksl_it it; + + for (it = ngtcp2_ksl_begin(&pktns->crypto.tx.frq); !ngtcp2_ksl_it_end(&it); + ngtcp2_ksl_it_next(&it)) { + frc = ngtcp2_ksl_it_get(&it); + ngtcp2_frame_chain_del(frc, conn->mem); + } + ngtcp2_ksl_clear(&pktns->crypto.tx.frq); +} + /* - * conn_cryptofrq_top returns the element which sits on top of the - * queue. The queue must not be empty. + * conn_cryptofrq_unacked_offset returns the CRYPTO frame offset by + * taking into account acknowledged offset. If there is no data to + * send, this function returns (uint64_t)-1. */ -static ngtcp2_frame_chain *conn_cryptofrq_top(ngtcp2_conn *conn, +static uint64_t conn_cryptofrq_unacked_offset(ngtcp2_conn *conn, ngtcp2_pktns *pktns) { + ngtcp2_frame_chain *frc; + ngtcp2_crypto *fr; + ngtcp2_range gap; + ngtcp2_rtb *rtb = &pktns->rtb; ngtcp2_ksl_it it; + size_t datalen; + (void)conn; - assert(ngtcp2_ksl_len(&pktns->crypto.tx.frq)); + for (it = ngtcp2_ksl_begin(&pktns->crypto.tx.frq); !ngtcp2_ksl_it_end(&it); + ngtcp2_ksl_it_next(&it)) { + frc = ngtcp2_ksl_it_get(&it); + fr = &frc->fr.crypto; + + gap = ngtcp2_strm_get_unacked_range_after(rtb->crypto, fr->offset); + + datalen = ngtcp2_vec_len(fr->data, fr->datacnt); + + if (gap.begin <= fr->offset) { + return fr->offset; + } + if (gap.begin < fr->offset + datalen) { + return gap.begin; + } + } - it = ngtcp2_ksl_begin(&pktns->crypto.tx.frq); - return ngtcp2_ksl_it_get(&it); + return (uint64_t)-1; } static int conn_cryptofrq_unacked_pop(ngtcp2_conn *conn, ngtcp2_pktns *pktns, @@ -1286,7 +1415,6 @@ static int conn_cryptofrq_unacked_pop(ngtcp2_conn *conn, ngtcp2_pktns *pktns, uint64_t offset, end_offset; size_t idx, end_idx; uint64_t base_offset, end_base_offset; - ngtcp2_ksl_it gapit; ngtcp2_range gap; ngtcp2_rtb *rtb = &pktns->rtb; ngtcp2_vec *v; @@ -1305,9 +1433,7 @@ static int conn_cryptofrq_unacked_pop(ngtcp2_conn *conn, ngtcp2_pktns *pktns, offset = fr->offset; base_offset = 0; - gapit = - ngtcp2_gaptr_get_first_gap_after(&rtb->crypto->tx.acked_offset, offset); - gap = *(ngtcp2_range *)ngtcp2_ksl_it_key(&gapit); + gap = ngtcp2_strm_get_unacked_range_after(rtb->crypto, offset); if (gap.begin < offset) { gap.begin = offset; } @@ -1370,6 +1496,7 @@ static int conn_cryptofrq_unacked_pop(ngtcp2_conn *conn, ngtcp2_pktns *pktns, } nfr = &nfrc->fr.crypto; + nfr->type = NGTCP2_FRAME_CRYPTO; memcpy(nfr->data, fr->data + end_idx, sizeof(nfr->data[0]) * (fr->datacnt - end_idx)); @@ -1707,6 +1834,8 @@ static ngtcp2_ssize conn_write_handshake_pkt(ngtcp2_conn *conn, uint8_t *dest, int pkt_empty = 1; int padded = 0; int hd_logged = 0; + uint64_t crypto_offset; + ngtcp2_ssize num_reclaimed; switch (type) { case NGTCP2_PKT_INITIAL: @@ -1731,7 +1860,7 @@ static ngtcp2_ssize conn_write_handshake_pkt(ngtcp2_conn *conn, uint8_t *dest, cc.aead = pktns->crypto.ctx.aead; cc.hp = pktns->crypto.ctx.hp; cc.ckm = pktns->crypto.tx.ckm; - cc.hp_key = pktns->crypto.tx.hp_key; + cc.hp_ctx = pktns->crypto.tx.hp_ctx; cc.encrypt = conn->callbacks.encrypt; cc.hp_mask = conn->callbacks.hp_mask; @@ -1757,7 +1886,7 @@ static ngtcp2_ssize conn_write_handshake_pkt(ngtcp2_conn *conn, uint8_t *dest, return 0; } - rv = conn_create_ack_frame(conn, &ackfr, &pktns->acktr, type, ts, + rv = conn_create_ack_frame(conn, &ackfr, pktns, type, ts, /* ack_delay = */ 0, NGTCP2_DEFAULT_ACK_DELAY_EXPONENT); if (rv != 0) { @@ -1776,11 +1905,17 @@ static ngtcp2_ssize conn_write_handshake_pkt(ngtcp2_conn *conn, uint8_t *dest, } } +build_pkt: for (; ngtcp2_ksl_len(&pktns->crypto.tx.frq);) { left = ngtcp2_ppe_left(&ppe); - left = ngtcp2_pkt_crypto_max_datalen( - conn_cryptofrq_top(conn, pktns)->fr.crypto.offset, left, left); + crypto_offset = conn_cryptofrq_unacked_offset(conn, pktns); + if (crypto_offset == (size_t)-1) { + conn_cryptofrq_clear(conn, pktns); + break; + } + + left = ngtcp2_pkt_crypto_max_datalen(crypto_offset, left, left); if (left == (size_t)-1) { break; } @@ -1806,7 +1941,33 @@ static ngtcp2_ssize conn_write_handshake_pkt(ngtcp2_conn *conn, uint8_t *dest, pkt_empty = 0; rtb_entry_flags |= - NGTCP2_RTB_FLAG_ACK_ELICITING | NGTCP2_RTB_FLAG_CRYPTO_PKT; + NGTCP2_RTB_FLAG_ACK_ELICITING | NGTCP2_RTB_FLAG_RETRANSMITTABLE; + } + + if (!(rtb_entry_flags & NGTCP2_RTB_FLAG_ACK_ELICITING) && + pktns->rtb.num_retransmittable && pktns->rtb.probe_pkt_left) { + num_reclaimed = ngtcp2_rtb_reclaim_on_pto(&pktns->rtb, conn, pktns, + pktns->rtb.probe_pkt_left + 1); + if (num_reclaimed < 0) { + ngtcp2_frame_chain_list_del(frq, conn->mem); + return rv; + } + if (num_reclaimed) { + goto build_pkt; + } + /* We had pktns->rtb.num_retransmittable > 0 but the contents of + those packets have been acknowledged (i.e., retransmission in + another packet). For server, in this case, we don't have to + send any probe packet. Client needs to send probe packets + until it knows that server has completed address validation or + handshake has been confirmed. */ + if (pktns->rtb.num_retransmittable == 0 && + (conn->server || + (conn->flags & (NGTCP2_CONN_FLAG_SERVER_ADDR_VERIFIED | + NGTCP2_CONN_FLAG_HANDSHAKE_CONFIRMED)))) { + pktns->rtb.probe_pkt_left = 0; + ngtcp2_conn_set_loss_detection_timer(conn, ts); + } } /* Don't send any PING frame if client Initial has not been @@ -1961,7 +2122,7 @@ static ngtcp2_ssize conn_write_ack_pkt(ngtcp2_conn *conn, uint8_t *dest, } ackfr = NULL; - rv = conn_create_ack_frame(conn, &ackfr, &pktns->acktr, type, ts, ack_delay, + rv = conn_create_ack_frame(conn, &ackfr, pktns, type, ts, ack_delay, ack_delay_exponent); if (rv != 0) { return rv; @@ -1990,6 +2151,11 @@ static void conn_discard_pktns(ngtcp2_conn *conn, ngtcp2_pktns **ppktns, conn->cstat.last_tx_pkt_ts[pktns->rtb.pktns_id] = UINT64_MAX; conn->cstat.loss_time[pktns->rtb.pktns_id] = UINT64_MAX; + conn_call_delete_crypto_aead_ctx(conn, &pktns->crypto.rx.ckm->aead_ctx); + conn_call_delete_crypto_cipher_ctx(conn, &pktns->crypto.rx.hp_ctx); + conn_call_delete_crypto_aead_ctx(conn, &pktns->crypto.tx.ckm->aead_ctx); + conn_call_delete_crypto_cipher_ctx(conn, &pktns->crypto.tx.hp_ctx); + pktns_del(pktns, conn->mem); *ppktns = NULL; @@ -2386,9 +2552,9 @@ typedef enum { /* NGTCP2_WRITE_PKT_FLAG_REQUIRE_PADDING indicates that packet should be padded */ NGTCP2_WRITE_PKT_FLAG_REQUIRE_PADDING = 0x01, - /* NGTCP2_WRITE_PKT_FLAG_STREAM_MORE indicates that more stream DATA - may come and it should be encoded into the current packet. */ - NGTCP2_WRITE_PKT_FLAG_STREAM_MORE = 0x02, + /* NGTCP2_WRITE_PKT_FLAG_MORE indicates that more frames might come + and it should be encoded into the current packet. */ + NGTCP2_WRITE_PKT_FLAG_MORE = 0x02, } ngtcp2_write_pkt_flag; /* @@ -2397,13 +2563,14 @@ typedef enum { * packet. It can be NGTCP2_PKT_SHORT or NGTCP2_PKT_0RTT. * * This function can send new stream data. In order to send stream - * data, specify the underlying stream to |data_strm|. If |fin| is - * set to nonzero, it signals that the given data is the final portion - * of the stream. |datav| vector of length |datavcnt| specify stream - * data to send. If no stream data to send, set |strm| to NULL. The - * number of bytes sent to the stream is assigned to |*pdatalen|. If - * 0 length STREAM data is sent, 0 is assigned to |*pdatalen|. The - * caller should initialize |*pdatalen| to -1. + * data, specify the underlying stream and parameters to + * |vmsg|->stream. If |vmsg|->stream.fin is set to nonzero, it + * signals that the given data is the final portion of the stream. + * |vmsg|->stream.data vector of length |vmsg|->stream.datacnt + * specifies stream data to send. The number of bytes sent to the + * stream is assigned to *|vmsg|->stream.pdatalen. If 0 length STREAM + * data is sent, 0 is assigned to it. The caller should initialize + * *|vmsg|->stream.pdatalen to -1. * * If |require_padding| is nonzero, padding bytes are added to occupy * the remaining packet payload. @@ -2419,10 +2586,8 @@ typedef enum { * Stream data could not be written because of flow control. */ static ngtcp2_ssize conn_write_pkt(ngtcp2_conn *conn, uint8_t *dest, - size_t destlen, ngtcp2_ssize *pdatalen, - uint8_t type, ngtcp2_strm *data_strm, - int fin, const ngtcp2_vec *datav, - size_t datavcnt, uint8_t flags, + size_t destlen, ngtcp2_vmsg *vmsg, + uint8_t type, uint8_t flags, ngtcp2_tstamp ts) { int rv = 0; ngtcp2_crypto_cc *cc = &conn->pkt.cc; @@ -2439,7 +2604,7 @@ static ngtcp2_ssize conn_write_pkt(ngtcp2_conn *conn, uint8_t *dest, int stream_blocked = 0; ngtcp2_pktns *pktns = &conn->pktns; size_t left; - size_t datalen = ngtcp2_vec_len(datav, datavcnt); + size_t datalen = 0; ngtcp2_vec data[NGTCP2_MAX_STREAM_DATACNT]; size_t datacnt; uint8_t rtb_entry_flags = NGTCP2_RTB_FLAG_NONE; @@ -2447,12 +2612,16 @@ static ngtcp2_ssize conn_write_pkt(ngtcp2_conn *conn, uint8_t *dest, ngtcp2_path_challenge_entry *pcent; uint8_t hd_flags; int require_padding = (flags & NGTCP2_WRITE_PKT_FLAG_REQUIRE_PADDING) != 0; - int stream_more = (flags & NGTCP2_WRITE_PKT_FLAG_STREAM_MORE) != 0; + int write_more = (flags & NGTCP2_WRITE_PKT_FLAG_MORE) != 0; int ppe_pending = (conn->flags & NGTCP2_CONN_FLAG_PPE_PENDING) != 0; size_t min_pktlen = conn_min_short_pktlen(conn); int padded = 0; int credit_expanded = 0; ngtcp2_cc_pkt cc_pkt; + uint64_t crypto_offset; + uint64_t stream_offset; + ngtcp2_ssize num_reclaimed; + int fin; /* Return 0 if destlen is less than minimum packet length which can trigger Stateless Reset */ @@ -2460,13 +2629,20 @@ static ngtcp2_ssize conn_write_pkt(ngtcp2_conn *conn, uint8_t *dest, return 0; } - if (data_strm) { - ndatalen = conn_enforce_flow_control(conn, data_strm, datalen); - /* 0 length STREAM frame is allowed */ - if (ndatalen || datalen == 0) { - send_stream = 1; - } else { - stream_blocked = 1; + if (vmsg) { + switch (vmsg->type) { + case NGTCP2_VMSG_TYPE_STREAM: + datalen = ngtcp2_vec_len(vmsg->stream.data, vmsg->stream.datacnt); + ndatalen = conn_enforce_flow_control(conn, vmsg->stream.strm, datalen); + /* 0 length STREAM frame is allowed */ + if (ndatalen || datalen == 0) { + send_stream = 1; + } else { + stream_blocked = 1; + } + break; + default: + break; } } @@ -2478,7 +2654,7 @@ static ngtcp2_ssize conn_write_pkt(ngtcp2_conn *conn, uint8_t *dest, ? NGTCP2_PKT_FLAG_KEY_PHASE : NGTCP2_PKT_FLAG_NONE; cc->ckm = pktns->crypto.tx.ckm; - cc->hp_key = pktns->crypto.tx.hp_key; + cc->hp_ctx = pktns->crypto.tx.hp_ctx; /* transport parameter is only valid after handshake completion which means we don't know how many connection ID that remote @@ -2499,7 +2675,7 @@ static ngtcp2_ssize conn_write_pkt(ngtcp2_conn *conn, uint8_t *dest, } hd_flags = NGTCP2_PKT_FLAG_LONG_FORM; cc->ckm = conn->early.ckm; - cc->hp_key = conn->early.hp_key; + cc->hp_ctx = conn->early.hp_ctx; break; default: /* Unreachable */ @@ -2566,7 +2742,7 @@ static ngtcp2_ssize conn_write_pkt(ngtcp2_conn *conn, uint8_t *dest, } rv = conn_create_ack_frame( - conn, &ackfr, &pktns->acktr, type, ts, conn_compute_ack_delay(conn), + conn, &ackfr, pktns, type, ts, conn_compute_ack_delay(conn), conn->local.settings.transport_params.ack_delay_exponent); if (rv != 0) { assert(ngtcp2_err_is_fatal(rv)); @@ -2585,7 +2761,16 @@ static ngtcp2_ssize conn_write_pkt(ngtcp2_conn *conn, uint8_t *dest, } } + build_pkt: for (pfrc = &pktns->tx.frq; *pfrc;) { + if ((*pfrc)->binder && + ((*pfrc)->binder->flags & NGTCP2_FRAME_CHAIN_BINDER_FLAG_ACK)) { + frc = *pfrc; + *pfrc = (*pfrc)->next; + ngtcp2_frame_chain_del(frc, conn->mem); + continue; + } + switch ((*pfrc)->fr.type) { case NGTCP2_FRAME_STOP_SENDING: strm = @@ -2649,7 +2834,8 @@ static ngtcp2_ssize conn_write_pkt(ngtcp2_conn *conn, uint8_t *dest, } pkt_empty = 0; - rtb_entry_flags |= NGTCP2_RTB_FLAG_ACK_ELICITING; + rtb_entry_flags |= + NGTCP2_RTB_FLAG_ACK_ELICITING | NGTCP2_RTB_FLAG_RETRANSMITTABLE; pfrc = &(*pfrc)->next; } @@ -2657,8 +2843,13 @@ static ngtcp2_ssize conn_write_pkt(ngtcp2_conn *conn, uint8_t *dest, for (; ngtcp2_ksl_len(&pktns->crypto.tx.frq);) { left = ngtcp2_ppe_left(ppe); - left = ngtcp2_pkt_crypto_max_datalen( - conn_cryptofrq_top(conn, pktns)->fr.crypto.offset, left, left); + crypto_offset = conn_cryptofrq_unacked_offset(conn, pktns); + if (crypto_offset == (size_t)-1) { + conn_cryptofrq_clear(conn, pktns); + break; + } + + left = ngtcp2_pkt_crypto_max_datalen(crypto_offset, left, left); if (left == (size_t)-1) { break; @@ -2683,7 +2874,8 @@ static ngtcp2_ssize conn_write_pkt(ngtcp2_conn *conn, uint8_t *dest, pfrc = &(*pfrc)->next; pkt_empty = 0; - rtb_entry_flags |= NGTCP2_RTB_FLAG_ACK_ELICITING; + rtb_entry_flags |= + NGTCP2_RTB_FLAG_ACK_ELICITING | NGTCP2_RTB_FLAG_RETRANSMITTABLE; } } @@ -2714,7 +2906,8 @@ static ngtcp2_ssize conn_write_pkt(ngtcp2_conn *conn, uint8_t *dest, assert(NGTCP2_ERR_NOBUF == rv); } else { pkt_empty = 0; - rtb_entry_flags |= NGTCP2_RTB_FLAG_ACK_ELICITING; + rtb_entry_flags |= + NGTCP2_RTB_FLAG_ACK_ELICITING | NGTCP2_RTB_FLAG_RETRANSMITTABLE; pfrc = &(*pfrc)->next; } } @@ -2745,7 +2938,8 @@ static ngtcp2_ssize conn_write_pkt(ngtcp2_conn *conn, uint8_t *dest, assert(NGTCP2_ERR_NOBUF == rv); } else { pkt_empty = 0; - rtb_entry_flags |= NGTCP2_RTB_FLAG_ACK_ELICITING; + rtb_entry_flags |= + NGTCP2_RTB_FLAG_ACK_ELICITING | NGTCP2_RTB_FLAG_RETRANSMITTABLE; pfrc = &(*pfrc)->next; } } @@ -2776,7 +2970,8 @@ static ngtcp2_ssize conn_write_pkt(ngtcp2_conn *conn, uint8_t *dest, pkt_empty = 0; credit_expanded = 1; - rtb_entry_flags |= NGTCP2_RTB_FLAG_ACK_ELICITING; + rtb_entry_flags |= + NGTCP2_RTB_FLAG_ACK_ELICITING | NGTCP2_RTB_FLAG_RETRANSMITTABLE; pfrc = &(*pfrc)->next; strm->rx.max_offset = strm->rx.unsent_max_offset; } @@ -2786,11 +2981,17 @@ static ngtcp2_ssize conn_write_pkt(ngtcp2_conn *conn, uint8_t *dest, continue; } + stream_offset = ngtcp2_strm_streamfrq_unacked_offset(strm); + if (stream_offset == (uint64_t)-1) { + ngtcp2_strm_streamfrq_clear(strm); + ngtcp2_conn_tx_strmq_pop(conn); + continue; + } + left = ngtcp2_ppe_left(ppe); - left = ngtcp2_pkt_stream_max_datalen( - strm->stream_id, ngtcp2_strm_streamfrq_top(strm)->fr.stream.offset, - left, left); + left = ngtcp2_pkt_stream_max_datalen(strm->stream_id, stream_offset, + left, left); if (left == (size_t)-1) { break; @@ -2816,7 +3017,8 @@ static ngtcp2_ssize conn_write_pkt(ngtcp2_conn *conn, uint8_t *dest, pfrc = &(*pfrc)->next; pkt_empty = 0; - rtb_entry_flags |= NGTCP2_RTB_FLAG_ACK_ELICITING; + rtb_entry_flags |= + NGTCP2_RTB_FLAG_ACK_ELICITING | NGTCP2_RTB_FLAG_RETRANSMITTABLE; if (ngtcp2_strm_streamfrq_empty(strm)) { ngtcp2_conn_tx_strmq_pop(conn); @@ -2837,7 +3039,7 @@ static ngtcp2_ssize conn_write_pkt(ngtcp2_conn *conn, uint8_t *dest, decrease packet count. */ if (ackfr == NULL && credit_expanded) { rv = conn_create_ack_frame( - conn, &ackfr, &pktns->acktr, type, ts, /* ack_delay = */ 0, + conn, &ackfr, pktns, type, ts, /* ack_delay = */ 0, conn->local.settings.transport_params.ack_delay_exponent); if (rv != 0) { assert(ngtcp2_err_is_fatal(rv)); @@ -2855,6 +3057,29 @@ static ngtcp2_ssize conn_write_pkt(ngtcp2_conn *conn, uint8_t *dest, } } } + + if (rv != NGTCP2_ERR_NOBUF && !send_stream && + !(rtb_entry_flags & NGTCP2_RTB_FLAG_ACK_ELICITING) && + pktns->rtb.num_retransmittable && pktns->tx.frq == NULL && + pktns->rtb.probe_pkt_left) { + num_reclaimed = ngtcp2_rtb_reclaim_on_pto(&pktns->rtb, conn, pktns, + pktns->rtb.probe_pkt_left + 1); + if (num_reclaimed < 0) { + return rv; + } + if (num_reclaimed) { + goto build_pkt; + } + + /* We had pktns->rtb.num_retransmittable > 0 but the contents of + those packets have been acknowledged (i.e., retransmission in + another packet). In this case, we don't have to send any + probe packet. */ + if (pktns->rtb.num_retransmittable == 0) { + pktns->rtb.probe_pkt_left = 0; + ngtcp2_conn_set_loss_detection_timer(conn, ts); + } + } } else { pfrc = conn->pkt.pfrc; rtb_entry_flags |= conn->pkt.rtb_entry_flags; @@ -2865,12 +3090,13 @@ static ngtcp2_ssize conn_write_pkt(ngtcp2_conn *conn, uint8_t *dest, left = ngtcp2_ppe_left(ppe); if (rv != NGTCP2_ERR_NOBUF && send_stream && *pfrc == NULL && - (ndatalen = ngtcp2_pkt_stream_max_datalen(data_strm->stream_id, - data_strm->tx.offset, ndatalen, - left)) != (size_t)-1 && + (ndatalen = ngtcp2_pkt_stream_max_datalen( + vmsg->stream.strm->stream_id, vmsg->stream.strm->tx.offset, ndatalen, + left)) != (size_t)-1 && (ndatalen || datalen == 0)) { datacnt = ngtcp2_vec_copy_at_most( - data, &ndatalen, NGTCP2_MAX_STREAM_DATACNT, datav, datavcnt, ndatalen); + data, &ndatalen, NGTCP2_MAX_STREAM_DATACNT, vmsg->stream.data, + vmsg->stream.datacnt, ndatalen); rv = ngtcp2_frame_chain_stream_datacnt_new(&nfrc, datacnt, conn->mem); if (rv != 0) { @@ -2880,12 +3106,13 @@ static ngtcp2_ssize conn_write_pkt(ngtcp2_conn *conn, uint8_t *dest, nfrc->fr.stream.type = NGTCP2_FRAME_STREAM; nfrc->fr.stream.flags = 0; - nfrc->fr.stream.stream_id = data_strm->stream_id; - nfrc->fr.stream.offset = data_strm->tx.offset; + nfrc->fr.stream.stream_id = vmsg->stream.strm->stream_id; + nfrc->fr.stream.offset = vmsg->stream.strm->tx.offset; nfrc->fr.stream.datacnt = datacnt; ngtcp2_vec_copy(nfrc->fr.stream.data, data, datacnt); - fin = fin && ndatalen == datalen; + fin = (vmsg->stream.flags & NGTCP2_WRITE_STREAM_FLAG_FIN) && + ndatalen == datalen; nfrc->fr.stream.fin = (uint8_t)fin; rv = conn_ppe_write_frame_hd_log(conn, ppe, &hd_logged, hd, &nfrc->fr); @@ -2897,17 +3124,18 @@ static ngtcp2_ssize conn_write_pkt(ngtcp2_conn *conn, uint8_t *dest, pfrc = &(*pfrc)->next; pkt_empty = 0; - rtb_entry_flags |= NGTCP2_RTB_FLAG_ACK_ELICITING; + rtb_entry_flags |= + NGTCP2_RTB_FLAG_ACK_ELICITING | NGTCP2_RTB_FLAG_RETRANSMITTABLE; - data_strm->tx.offset += ndatalen; + vmsg->stream.strm->tx.offset += ndatalen; conn->tx.offset += ndatalen; if (fin) { - ngtcp2_strm_shutdown(data_strm, NGTCP2_STRM_FLAG_SHUT_WR); + ngtcp2_strm_shutdown(vmsg->stream.strm, NGTCP2_STRM_FLAG_SHUT_WR); } - if (pdatalen) { - *pdatalen = (ngtcp2_ssize)ndatalen; + if (vmsg->stream.pdatalen) { + *vmsg->stream.pdatalen = (ngtcp2_ssize)ndatalen; } } else { send_stream = 0; @@ -2922,7 +3150,7 @@ static ngtcp2_ssize conn_write_pkt(ngtcp2_conn *conn, uint8_t *dest, if (conn->pktns.rtb.probe_pkt_left == 0) { return 0; } - } else if (stream_more) { + } else if (write_more) { conn->pkt.pfrc = pfrc; conn->pkt.pkt_empty = pkt_empty; conn->pkt.rtb_entry_flags = rtb_entry_flags; @@ -2930,7 +3158,7 @@ static ngtcp2_ssize conn_write_pkt(ngtcp2_conn *conn, uint8_t *dest, conn->flags |= NGTCP2_CONN_FLAG_PPE_PENDING; if (send_stream) { - return NGTCP2_ERR_WRITE_STREAM_MORE; + return NGTCP2_ERR_WRITE_MORE; } if (ngtcp2_conn_get_max_data_left(conn) && stream_blocked) { @@ -2939,8 +3167,7 @@ static ngtcp2_ssize conn_write_pkt(ngtcp2_conn *conn, uint8_t *dest, } if (!(rtb_entry_flags & NGTCP2_RTB_FLAG_ACK_ELICITING)) { - if (pktns->rtb.probe_pkt_left || - pktns->tx.num_non_ack_pkt >= NGTCP2_MAX_NON_ACK_TX_PKT) { + if (pktns->tx.num_non_ack_pkt >= NGTCP2_MAX_NON_ACK_TX_PKT) { lfr.type = NGTCP2_FRAME_PING; rv = conn_ppe_write_frame_hd_log(conn, ppe, &hd_logged, hd, &lfr); @@ -2950,9 +3177,6 @@ static ngtcp2_ssize conn_write_pkt(ngtcp2_conn *conn, uint8_t *dest, packet is still empty. */ } else { rtb_entry_flags |= NGTCP2_RTB_FLAG_ACK_ELICITING; - if (pktns->rtb.probe_pkt_left) { - rtb_entry_flags |= NGTCP2_RTB_FLAG_PROBE; - } pktns->tx.num_non_ack_pkt = 0; } } else { @@ -2966,6 +3190,8 @@ static ngtcp2_ssize conn_write_pkt(ngtcp2_conn *conn, uint8_t *dest, before ngtcp2_rtb_entry is safely created and added. */ lfr.type = NGTCP2_FRAME_PADDING; if ((require_padding || + /* Making full sized packet will help GSO a bit */ + ngtcp2_ppe_left(ppe) < 10 || (type == NGTCP2_PKT_0RTT && conn->state == NGTCP2_CS_CLIENT_INITIAL)) && ngtcp2_ppe_left(ppe)) { lfr.padding.len = ngtcp2_ppe_padding(ppe); @@ -2989,6 +3215,8 @@ static ngtcp2_ssize conn_write_pkt(ngtcp2_conn *conn, uint8_t *dest, ngtcp2_qlog_pkt_sent_end(&conn->qlog, hd, (size_t)nwrite); + /* TODO ack-eliciting vs needs-tracking */ + /* probe packet needs tracking but it does not need ACK, could be lost. */ if ((rtb_entry_flags & NGTCP2_RTB_FLAG_ACK_ELICITING) || padded) { rv = ngtcp2_rtb_entry_new(&ent, hd, NULL, ts, (size_t)nwrite, rtb_entry_flags, conn->mem); @@ -3090,7 +3318,7 @@ ngtcp2_conn_write_single_frame_pkt(ngtcp2_conn *conn, uint8_t *dest, cc.aead = pktns->crypto.ctx.aead; cc.hp = pktns->crypto.ctx.hp; cc.ckm = pktns->crypto.tx.ckm; - cc.hp_key = pktns->crypto.tx.hp_key; + cc.hp_ctx = pktns->crypto.tx.hp_ctx; cc.encrypt = conn->callbacks.encrypt; cc.hp_mask = conn->callbacks.hp_mask; @@ -3206,9 +3434,9 @@ static int conn_handshake_remnants_left(ngtcp2_conn *conn) { ngtcp2_pktns *hs_pktns = conn->hs_pktns; return !(conn->flags & NGTCP2_CONN_FLAG_HANDSHAKE_COMPLETED) || - (in_pktns && (in_pktns->rtb.num_ack_eliciting || + (in_pktns && (in_pktns->rtb.num_retransmittable || ngtcp2_ksl_len(&in_pktns->crypto.tx.frq))) || - (hs_pktns && (hs_pktns->rtb.num_ack_eliciting || + (hs_pktns && (hs_pktns->rtb.num_retransmittable || ngtcp2_ksl_len(&hs_pktns->crypto.tx.frq))); } @@ -3388,9 +3616,9 @@ static ngtcp2_ssize conn_write_path_challenge(ngtcp2_conn *conn, } assert(conn->callbacks.rand); - rv = conn->callbacks.rand(conn, lfr.path_challenge.data, + rv = conn->callbacks.rand(lfr.path_challenge.data, sizeof(lfr.path_challenge.data), - NGTCP2_RAND_CTX_PATH_CHALLENGE, conn->user_data); + NGTCP2_RAND_CTX_PATH_CHALLENGE); if (rv != 0) { return NGTCP2_ERR_CALLBACK_FAILURE; } @@ -3399,7 +3627,7 @@ static ngtcp2_ssize conn_write_path_challenge(ngtcp2_conn *conn, /* TODO reconsider this. This might get larger pretty quickly than validation timeout which is just around 3*PTO. */ - expiry = ts + 6ULL * NGTCP2_DEFAULT_INITIAL_RTT * (1ULL << pv->loss_count); + expiry = ts + 6ULL * conn->cstat.initial_rtt * (1ULL << pv->loss_count); ngtcp2_pv_add_entry(pv, lfr.path_challenge.data, expiry); @@ -3505,17 +3733,18 @@ static uint64_t conn_tx_strmq_first_cycle(ngtcp2_conn *conn) { return strm->cycle; } -/* - * conn_resched_frames reschedules frames linked from |*pfrc| for - * retransmission. - * - * This function returns 0 if it succeeds, or one of the following - * negative error codes: - * - * NGTCP2_ERR_NOMEM - * Out of memory. - */ -static int conn_resched_frames(ngtcp2_conn *conn, ngtcp2_pktns *pktns, +uint64_t ngtcp2_conn_tx_strmq_first_cycle(ngtcp2_conn *conn) { + ngtcp2_strm *strm; + + if (ngtcp2_pq_empty(&conn->tx.strmq)) { + return 0; + } + + strm = ngtcp2_struct_of(ngtcp2_pq_top(&conn->tx.strmq), ngtcp2_strm, pe); + return strm->cycle; +} + +int ngtcp2_conn_resched_frames(ngtcp2_conn *conn, ngtcp2_pktns *pktns, ngtcp2_frame_chain **pfrc) { ngtcp2_frame_chain **first = pfrc; ngtcp2_frame_chain *frc; @@ -3605,7 +3834,6 @@ static int conn_on_retry(ngtcp2_conn *conn, const ngtcp2_pkt_hd *hd, ngtcp2_rtb *rtb = &conn->pktns.rtb; ngtcp2_rtb *in_rtb; uint8_t cidbuf[sizeof(retry.odcid.data) * 2 + 1]; - ngtcp2_frame_chain *frc = NULL; ngtcp2_vec *token; if (!in_pktns || conn->flags & NGTCP2_CONN_FLAG_RECV_RETRY) { @@ -3622,7 +3850,8 @@ static int conn_on_retry(ngtcp2_conn *conn, const ngtcp2_pkt_hd *hd, retry.odcid = conn->dcid.current.cid; rv = ngtcp2_pkt_verify_retry_tag(&retry, pkt, pktlen, conn->callbacks.encrypt, - &conn->crypto.retry_aead); + &conn->crypto.retry_aead, + &conn->crypto.retry_aead_ctx); if (rv != 0) { ngtcp2_log_info(&conn->log, NGTCP2_LOG_EVENT_PKT, "unable to verify Retry packet integrity"); @@ -3659,22 +3888,13 @@ static int conn_on_retry(ngtcp2_conn *conn, const ngtcp2_pkt_hd *hd, /* Just freeing memory is dangerous because we might free twice. */ - ngtcp2_rtb_remove_all(rtb, &frc, &conn->cstat); - - rv = conn_resched_frames(conn, &conn->pktns, &frc); + rv = ngtcp2_rtb_remove_all(rtb, conn, &conn->pktns, &conn->cstat); if (rv != 0) { - assert(ngtcp2_err_is_fatal(rv)); - ngtcp2_frame_chain_list_del(frc, conn->mem); return rv; } - frc = NULL; - ngtcp2_rtb_remove_all(in_rtb, &frc, &conn->cstat); - - rv = conn_resched_frames(conn, in_pktns, &frc); + rv = ngtcp2_rtb_remove_all(in_rtb, conn, in_pktns, &conn->cstat); if (rv != 0) { - assert(ngtcp2_err_is_fatal(rv)); - ngtcp2_frame_chain_list_del(frc, conn->mem); return rv; } @@ -3692,6 +3912,7 @@ static int conn_on_retry(ngtcp2_conn *conn, const ngtcp2_pkt_hd *hd, ngtcp2_cpymem(token->base, retry.token.base, retry.token.len); + reset_conn_stat_recovery(&conn->cstat); conn_reset_congestion_state(conn); return 0; @@ -3699,15 +3920,11 @@ static int conn_on_retry(ngtcp2_conn *conn, const ngtcp2_pkt_hd *hd, int ngtcp2_conn_detect_lost_pkt(ngtcp2_conn *conn, ngtcp2_pktns *pktns, ngtcp2_conn_stat *cstat, ngtcp2_tstamp ts) { - ngtcp2_frame_chain *frc = NULL; int rv; ngtcp2_duration pto = conn_compute_pto(conn, pktns); - ngtcp2_rtb_detect_lost_pkt(&pktns->rtb, &frc, cstat, pto, ts); - - rv = conn_resched_frames(conn, pktns, &frc); + rv = ngtcp2_rtb_detect_lost_pkt(&pktns->rtb, conn, pktns, cstat, pto, ts); if (rv != 0) { - ngtcp2_frame_chain_list_del(frc, conn->mem); return rv; } @@ -3983,7 +4200,7 @@ static ngtcp2_ssize decrypt_pkt(uint8_t *dest, const ngtcp2_crypto_aead *aead, ngtcp2_crypto_create_nonce(nonce, ckm->iv.base, ckm->iv.len, pkt_num); - rv = decrypt(dest, aead, payload, payloadlen, ckm->key.base, nonce, + rv = decrypt(dest, aead, &ckm->aead_ctx, payload, payloadlen, nonce, ckm->iv.len, ad, adlen); if (rv != 0) { @@ -4017,7 +4234,7 @@ static ngtcp2_ssize decrypt_hp(ngtcp2_pkt_hd *hd, uint8_t *dest, size_t destlen, const ngtcp2_crypto_cipher *hp, const uint8_t *pkt, size_t pktlen, size_t pkt_num_offset, ngtcp2_crypto_km *ckm, - const ngtcp2_vec *hp_key, + const ngtcp2_crypto_cipher_ctx *hp_ctx, ngtcp2_hp_mask hp_mask) { size_t sample_offset; uint8_t *p = dest; @@ -4037,7 +4254,7 @@ static ngtcp2_ssize decrypt_hp(ngtcp2_pkt_hd *hd, uint8_t *dest, size_t destlen, sample_offset = pkt_num_offset + 4; - rv = hp_mask(mask, hp, hp_key->base, pkt + sample_offset); + rv = hp_mask(mask, hp, hp_ctx, pkt + sample_offset); if (rv != 0) { return NGTCP2_ERR_CALLBACK_FAILURE; } @@ -4083,8 +4300,12 @@ static int conn_emit_pending_crypto_data(ngtcp2_conn *conn, int rv; uint64_t offset; + if (!strm->rx.rob) { + return 0; + } + for (;;) { - datalen = ngtcp2_rob_data_at(&strm->rx.rob, &data, rx_offset); + datalen = ngtcp2_rob_data_at(strm->rx.rob, &data, rx_offset); if (datalen == 0) { assert(rx_offset == ngtcp2_strm_rx_offset(strm)); return 0; @@ -4098,7 +4319,7 @@ static int conn_emit_pending_crypto_data(ngtcp2_conn *conn, return rv; } - ngtcp2_rob_pop(&strm->rx.rob, rx_offset - datalen, datalen); + ngtcp2_rob_pop(strm->rx.rob, rx_offset - datalen, datalen); } } @@ -4129,7 +4350,7 @@ static void conn_recv_path_challenge(ngtcp2_conn *conn, * conn_reset_congestion_state resets congestion state. */ static void conn_reset_congestion_state(ngtcp2_conn *conn) { - conn_reset_conn_stat(conn, &conn->cstat); + conn_reset_conn_stat_cc(conn, &conn->cstat); conn_reset_rx_rate(conn); @@ -4210,16 +4431,16 @@ static int pktns_pkt_num_is_duplicate(ngtcp2_pktns *pktns, int64_t pkt_num) { * pktns_commit_recv_pkt_num marks packet number |pkt_num| as * received. */ -static int pktns_commit_recv_pkt_num(ngtcp2_pktns *pktns, int64_t pkt_num) { +static int pktns_commit_recv_pkt_num(ngtcp2_pktns *pktns, int64_t pkt_num, + ngtcp2_tstamp ts) { int rv; - ngtcp2_ksl_it it; - ngtcp2_range range; if (pktns->rx.max_pkt_num + 1 != pkt_num) { ngtcp2_acktr_immediate_ack(&pktns->acktr); } if (pktns->rx.max_pkt_num < pkt_num) { pktns->rx.max_pkt_num = pkt_num; + pktns->rx.max_pkt_ts = ts; } rv = ngtcp2_gaptr_push(&pktns->rx.pngap, (uint64_t)pkt_num, 1); @@ -4228,9 +4449,7 @@ static int pktns_commit_recv_pkt_num(ngtcp2_pktns *pktns, int64_t pkt_num) { } if (ngtcp2_ksl_len(&pktns->rx.pngap.gap) > 256) { - it = ngtcp2_ksl_begin(&pktns->rx.pngap.gap); - range = *(ngtcp2_range *)ngtcp2_ksl_it_key(&it); - ngtcp2_ksl_remove(&pktns->rx.pngap.gap, NULL, &range); + ngtcp2_gaptr_drop_first_gap(&pktns->rx.pngap); } return 0; @@ -4310,7 +4529,7 @@ static ngtcp2_ssize conn_recv_handshake_pkt(ngtcp2_conn *conn, ngtcp2_crypto_aead *aead; ngtcp2_crypto_cipher *hp; ngtcp2_crypto_km *ckm; - const ngtcp2_vec *hp_key; + ngtcp2_crypto_cipher_ctx *hp_ctx; ngtcp2_hp_mask hp_mask; ngtcp2_decrypt decrypt; size_t aead_overhead; @@ -4539,14 +4758,14 @@ static ngtcp2_ssize conn_recv_handshake_pkt(ngtcp2_conn *conn, aead = &pktns->crypto.ctx.aead; hp = &pktns->crypto.ctx.hp; ckm = pktns->crypto.rx.ckm; - hp_key = pktns->crypto.rx.hp_key; + hp_ctx = &pktns->crypto.rx.hp_ctx; assert(ckm); assert(hp_mask); assert(decrypt); nwrite = decrypt_hp(&hd, plain_hdpkt, sizeof(plain_hdpkt), hp, pkt, pktlen, - (size_t)nread, ckm, hp_key, hp_mask); + (size_t)nread, ckm, hp_ctx, hp_mask); if (nwrite < 0) { if (ngtcp2_err_is_fatal((int)nwrite)) { return nwrite; @@ -4712,7 +4931,7 @@ static ngtcp2_ssize conn_recv_handshake_pkt(ngtcp2_conn *conn, ngtcp2_qlog_pkt_received_end(&conn->qlog, &hd, pktlen); - rv = pktns_commit_recv_pkt_num(pktns, hd.pkt_num); + rv = pktns_commit_recv_pkt_num(pktns, hd.pkt_num, pkt_ts); if (rv != 0) { return rv; } @@ -4886,6 +5105,10 @@ static int conn_emit_pending_stream_data(ngtcp2_conn *conn, ngtcp2_strm *strm, uint32_t sdflags; int handshake_completed = conn->flags & NGTCP2_CONN_FLAG_HANDSHAKE_COMPLETED; + if (!strm->rx.rob) { + return 0; + } + for (;;) { /* Stop calling callback if application has called ngtcp2_conn_shutdown_stream_read() inside the callback. @@ -4894,7 +5117,7 @@ static int conn_emit_pending_stream_data(ngtcp2_conn *conn, ngtcp2_strm *strm, return 0; } - datalen = ngtcp2_rob_data_at(&strm->rx.rob, &data, rx_offset); + datalen = ngtcp2_rob_data_at(strm->rx.rob, &data, rx_offset); if (datalen == 0) { assert(rx_offset == ngtcp2_strm_rx_offset(strm)); return 0; @@ -4917,12 +5140,10 @@ static int conn_emit_pending_stream_data(ngtcp2_conn *conn, ngtcp2_strm *strm, return rv; } - ngtcp2_rob_pop(&strm->rx.rob, rx_offset - datalen, datalen); + ngtcp2_rob_pop(strm->rx.rob, rx_offset - datalen, datalen); } } -static int conn_on_crypto_timeout(ngtcp2_conn *conn, ngtcp2_pktns *pktns); - /* * conn_recv_crypto is called when CRYPTO frame |fr| is received. * |rx_offset_base| is the offset in the entire TLS handshake stream. @@ -4975,16 +5196,7 @@ static int conn_recv_crypto(ngtcp2_conn *conn, ngtcp2_crypto_level crypto_level, send a packet containing unacknowledged CRYPTO data earlier than the PTO expiry, subject to address validation limits; ... */ - rv = conn_on_crypto_timeout(conn, conn->in_pktns); - if (rv != 0) { - return rv; - } conn->in_pktns->rtb.probe_pkt_left = 1; - - rv = conn_on_crypto_timeout(conn, conn->hs_pktns); - if (rv != 0) { - return rv; - } conn->hs_pktns->rtb.probe_pkt_left = 1; } return 0; @@ -5004,7 +5216,7 @@ static int conn_recv_crypto(ngtcp2_conn *conn, ngtcp2_crypto_level crypto_level, uint64_t offset = rx_offset; rx_offset += datalen; - rv = ngtcp2_rob_remove_prefix(&crypto->rx.rob, rx_offset); + rv = ngtcp2_strm_update_rx_offset(crypto, rx_offset); if (rv != 0) { return rv; } @@ -5216,7 +5428,7 @@ static int conn_recv_stream(ngtcp2_conn *conn, const ngtcp2_stream *fr, datalen -= ncut; rx_offset += datalen; - rv = ngtcp2_rob_remove_prefix(&strm->rx.rob, rx_offset); + rv = ngtcp2_strm_update_rx_offset(strm, rx_offset); if (rv != 0) { return rv; } @@ -5803,39 +6015,53 @@ static int conn_recv_new_connection_id(ngtcp2_conn *conn, return 0; } - if (!found) { - len = ngtcp2_ringbuf_len(&conn->dcid.unused); + if (found) { + return 0; + } + + if (ngtcp2_gaptr_is_pushed(&conn->dcid.seqgap, fr->seq, 1)) { + return 0; + } + + rv = ngtcp2_gaptr_push(&conn->dcid.seqgap, fr->seq, 1); + if (rv != 0) { + return rv; + } + + if (ngtcp2_ksl_len(&conn->dcid.seqgap.gap) > 32) { + ngtcp2_gaptr_drop_first_gap(&conn->dcid.seqgap); + } + + len = ngtcp2_ringbuf_len(&conn->dcid.unused); - if (conn->dcid.current.seq >= conn->dcid.retire_prior_to) { + if (conn->dcid.current.seq >= conn->dcid.retire_prior_to) { + ++extra_dcid; + } + if (pv) { + if (pv->dcid.seq != conn->dcid.current.seq && + pv->dcid.seq >= conn->dcid.retire_prior_to) { ++extra_dcid; } - if (pv) { - if (pv->dcid.seq != conn->dcid.current.seq && - pv->dcid.seq >= conn->dcid.retire_prior_to) { - ++extra_dcid; - } - if ((pv->flags & NGTCP2_PV_FLAG_FALLBACK_ON_FAILURE) && - pv->fallback_dcid.seq != conn->dcid.current.seq && - pv->fallback_dcid.seq >= conn->dcid.retire_prior_to) { - ++extra_dcid; - } - } - - if (conn->local.settings.transport_params.active_connection_id_limit <= - len + extra_dcid) { - return NGTCP2_ERR_CONNECTION_ID_LIMIT; + if ((pv->flags & NGTCP2_PV_FLAG_FALLBACK_ON_FAILURE) && + pv->fallback_dcid.seq != conn->dcid.current.seq && + pv->fallback_dcid.seq >= conn->dcid.retire_prior_to) { + ++extra_dcid; } + } - if (len >= NGTCP2_MAX_DCID_POOL_SIZE) { - ngtcp2_log_info(&conn->log, NGTCP2_LOG_EVENT_CON, - "too many connection ID"); - return 0; - } + if (conn->local.settings.transport_params.active_connection_id_limit <= + len + extra_dcid) { + return NGTCP2_ERR_CONNECTION_ID_LIMIT; + } - dcid = ngtcp2_ringbuf_push_back(&conn->dcid.unused); - ngtcp2_dcid_init(dcid, fr->seq, &fr->cid, fr->stateless_reset_token); + if (len >= NGTCP2_MAX_DCID_POOL_SIZE) { + ngtcp2_log_info(&conn->log, NGTCP2_LOG_EVENT_CON, "too many connection ID"); + return 0; } + dcid = ngtcp2_ringbuf_push_back(&conn->dcid.unused); + ngtcp2_dcid_init(dcid, fr->seq, &fr->cid, fr->stateless_reset_token); + return 0; } @@ -6042,14 +6268,14 @@ static int conn_recv_new_token(ngtcp2_conn *conn, const ngtcp2_new_token *fr) { * User-defined callback function failed. */ static int conn_select_preferred_addr(ngtcp2_conn *conn) { - uint8_t buf[128]; + struct sockaddr_storage buf; ngtcp2_addr addr; int rv; ngtcp2_duration timeout; ngtcp2_pv *pv; ngtcp2_dcid *dcid; - ngtcp2_addr_init(&addr, buf, 0, NULL); + ngtcp2_addr_init(&addr, (struct sockaddr *)&buf, 0, NULL); if (ngtcp2_ringbuf_len(&conn->dcid.unused) == 0) { return 0; @@ -6069,8 +6295,7 @@ static int conn_select_preferred_addr(ngtcp2_conn *conn) { dcid = ngtcp2_ringbuf_get(&conn->dcid.unused, 0); timeout = 3 * conn_compute_pto(conn, &conn->pktns); - timeout = - ngtcp2_max(timeout, (ngtcp2_duration)(6ULL * NGTCP2_DEFAULT_INITIAL_RTT)); + timeout = ngtcp2_max(timeout, 6 * conn->cstat.initial_rtt); rv = ngtcp2_pv_new(&pv, dcid, timeout, NGTCP2_PV_FLAG_NONE, &conn->log, conn->mem); @@ -6116,8 +6341,17 @@ static int conn_recv_handshake_done(ngtcp2_conn *conn, ngtcp2_tstamp ts) { conn->flags |= NGTCP2_CONN_FLAG_HANDSHAKE_CONFIRMED | NGTCP2_CONN_FLAG_SERVER_ADDR_VERIFIED; + conn->pktns.rtb.persistent_congestion_start_ts = ts; + conn_discard_handshake_state(conn, ts); + if (conn->remote.transport_params.preferred_address_present) { + rv = conn_select_preferred_addr(conn); + if (rv != 0) { + return rv; + } + } + if (conn->callbacks.handshake_confirmed) { rv = conn->callbacks.handshake_confirmed(conn, conn->user_data); if (rv != 0) { @@ -6154,7 +6388,8 @@ static int conn_prepare_key_update(ngtcp2_conn *conn, ngtcp2_tstamp ts) { ngtcp2_crypto_km *rx_ckm = pktns->crypto.rx.ckm; ngtcp2_crypto_km *tx_ckm = pktns->crypto.tx.ckm; ngtcp2_crypto_km *new_rx_ckm, *new_tx_ckm; - size_t secretlen, keylen, ivlen; + ngtcp2_crypto_aead_ctx rx_aead_ctx = {0}, tx_aead_ctx = {0}; + size_t secretlen, ivlen; if ((conn->flags & NGTCP2_CONN_FLAG_HANDSHAKE_CONFIRMED) && (tx_ckm->use_count >= pktns->crypto.ctx.max_encryption || @@ -6175,17 +6410,16 @@ static int conn_prepare_key_update(ngtcp2_conn *conn, ngtcp2_tstamp ts) { } secretlen = rx_ckm->secret.len; - keylen = rx_ckm->key.len; ivlen = rx_ckm->iv.len; rv = ngtcp2_crypto_km_nocopy_new(&conn->crypto.key_update.new_rx_ckm, - secretlen, keylen, ivlen, conn->mem); + secretlen, ivlen, conn->mem); if (rv != 0) { return rv; } rv = ngtcp2_crypto_km_nocopy_new(&conn->crypto.key_update.new_tx_ckm, - secretlen, keylen, ivlen, conn->mem); + secretlen, ivlen, conn->mem); if (rv != 0) { return rv; } @@ -6196,21 +6430,27 @@ static int conn_prepare_key_update(ngtcp2_conn *conn, ngtcp2_tstamp ts) { assert(conn->callbacks.update_key); rv = conn->callbacks.update_key( - conn, new_rx_ckm->secret.base, new_tx_ckm->secret.base, - new_rx_ckm->key.base, new_rx_ckm->iv.base, new_tx_ckm->key.base, - new_tx_ckm->iv.base, rx_ckm->secret.base, tx_ckm->secret.base, secretlen, - conn->user_data); + conn, new_rx_ckm->secret.base, new_tx_ckm->secret.base, &rx_aead_ctx, + new_rx_ckm->iv.base, &tx_aead_ctx, new_tx_ckm->iv.base, + rx_ckm->secret.base, tx_ckm->secret.base, secretlen, conn->user_data); if (rv != 0) { return NGTCP2_ERR_CALLBACK_FAILURE; } + new_rx_ckm->aead_ctx = rx_aead_ctx; + new_tx_ckm->aead_ctx = tx_aead_ctx; + if (!(rx_ckm->flags & NGTCP2_CRYPTO_KM_FLAG_KEY_PHASE_ONE)) { new_rx_ckm->flags |= NGTCP2_CRYPTO_KM_FLAG_KEY_PHASE_ONE; new_tx_ckm->flags |= NGTCP2_CRYPTO_KM_FLAG_KEY_PHASE_ONE; } - ngtcp2_crypto_km_del(conn->crypto.key_update.old_rx_ckm, conn->mem); - conn->crypto.key_update.old_rx_ckm = NULL; + if (conn->crypto.key_update.old_rx_ckm) { + conn_call_delete_crypto_aead_ctx( + conn, &conn->crypto.key_update.old_rx_ckm->aead_ctx); + ngtcp2_crypto_km_del(conn->crypto.key_update.old_rx_ckm, conn->mem); + conn->crypto.key_update.old_rx_ckm = NULL; + } return 0; } @@ -6233,7 +6473,11 @@ static void conn_rotate_keys(ngtcp2_conn *conn, int64_t pkt_num) { conn->crypto.key_update.new_rx_ckm = NULL; pktns->crypto.rx.ckm->pkt_num = pkt_num; + assert(pktns->crypto.tx.ckm); + + conn_call_delete_crypto_aead_ctx(conn, &pktns->crypto.tx.ckm->aead_ctx); ngtcp2_crypto_km_del(pktns->crypto.tx.ckm, conn->mem); + pktns->crypto.tx.ckm = conn->crypto.key_update.new_tx_ckm; conn->crypto.key_update.new_tx_ckm = NULL; pktns->crypto.tx.ckm->pkt_num = pktns->tx.last_pkt_num + 1; @@ -6318,8 +6562,7 @@ static int conn_recv_non_probing_pkt_on_new_path(ngtcp2_conn *conn, "non-probing packet was received from new remote address"); timeout = 3 * conn_compute_pto(conn, &conn->pktns); - timeout = - ngtcp2_max(timeout, (ngtcp2_duration)(6ULL * NGTCP2_DEFAULT_INITIAL_RTT)); + timeout = ngtcp2_max(timeout, 6 * conn->cstat.initial_rtt); if (require_new_cid) { dcid = *(ngtcp2_dcid *)ngtcp2_ringbuf_get(&conn->dcid.unused, 0); @@ -6457,7 +6700,7 @@ conn_recv_delayed_handshake_pkt(ngtcp2_conn *conn, const ngtcp2_pkt_hd *hd, ngtcp2_qlog_pkt_received_end(&conn->qlog, hd, pktlen); - rv = pktns_commit_recv_pkt_num(pktns, hd->pkt_num); + rv = pktns_commit_recv_pkt_num(pktns, hd->pkt_num, pkt_ts); if (rv != 0) { return rv; } @@ -6466,7 +6709,8 @@ conn_recv_delayed_handshake_pkt(ngtcp2_conn *conn, const ngtcp2_pkt_hd *hd, ngtcp2_acktr_immediate_ack(&pktns->acktr); } - rv = ngtcp2_conn_sched_ack(conn, &pktns->acktr, hd->pkt_num, require_ack, ts); + rv = ngtcp2_conn_sched_ack(conn, &pktns->acktr, hd->pkt_num, require_ack, + pkt_ts); if (rv != 0) { return rv; } @@ -6529,7 +6773,7 @@ static ngtcp2_ssize conn_recv_pkt(ngtcp2_conn *conn, const ngtcp2_path *path, ngtcp2_crypto_aead *aead; ngtcp2_crypto_cipher *hp; ngtcp2_crypto_km *ckm; - const ngtcp2_vec *hp_key; + ngtcp2_crypto_cipher_ctx *hp_ctx; uint8_t plain_hdpkt[1500]; ngtcp2_hp_mask hp_mask; ngtcp2_decrypt decrypt; @@ -6578,7 +6822,7 @@ static ngtcp2_ssize conn_recv_pkt(ngtcp2_conn *conn, const ngtcp2_path *path, pktns = conn->hs_pktns; ckm = pktns->crypto.rx.ckm; - hp_key = pktns->crypto.rx.hp_key; + hp_ctx = &pktns->crypto.rx.hp_ctx; hp_mask = conn->callbacks.hp_mask; decrypt = conn->callbacks.decrypt; aead_overhead = conn->crypto.aead_overhead; @@ -6590,7 +6834,7 @@ static ngtcp2_ssize conn_recv_pkt(ngtcp2_conn *conn, const ngtcp2_path *path, pktns = &conn->pktns; ckm = conn->early.ckm; - hp_key = conn->early.hp_key; + hp_ctx = &conn->early.hp_ctx; hp_mask = conn->callbacks.hp_mask; decrypt = conn->callbacks.decrypt; aead_overhead = conn->crypto.aead_overhead; @@ -6611,7 +6855,7 @@ static ngtcp2_ssize conn_recv_pkt(ngtcp2_conn *conn, const ngtcp2_path *path, pktns = &conn->pktns; ckm = pktns->crypto.rx.ckm; - hp_key = pktns->crypto.rx.hp_key; + hp_ctx = &pktns->crypto.rx.hp_ctx; hp_mask = conn->callbacks.hp_mask; decrypt = conn->callbacks.decrypt; aead_overhead = conn->crypto.aead_overhead; @@ -6621,7 +6865,7 @@ static ngtcp2_ssize conn_recv_pkt(ngtcp2_conn *conn, const ngtcp2_path *path, hp = &pktns->crypto.ctx.hp; nwrite = decrypt_hp(&hd, plain_hdpkt, sizeof(plain_hdpkt), hp, pkt, pktlen, - (size_t)nread, ckm, hp_key, hp_mask); + (size_t)nread, ckm, hp_ctx, hp_mask); if (nwrite < 0) { if (ngtcp2_err_is_fatal((int)nwrite)) { return nwrite; @@ -6993,7 +7237,7 @@ static ngtcp2_ssize conn_recv_pkt(ngtcp2_conn *conn, const ngtcp2_path *path, } } - rv = pktns_commit_recv_pkt_num(pktns, hd.pkt_num); + rv = pktns_commit_recv_pkt_num(pktns, hd.pkt_num, pkt_ts); if (rv != 0) { return rv; } @@ -7287,8 +7531,9 @@ static int conn_read_handshake(ngtcp2_conn *conn, const ngtcp2_path *path, * packet buffered, perform address validation in order to buffer * validated data only. */ - if (ngtcp2_rob_first_gap_offset(&conn->in_pktns->crypto.strm.rx.rob) == 0) { - if (ngtcp2_rob_data_buffered(&conn->in_pktns->crypto.strm.rx.rob)) { + if (ngtcp2_strm_rx_offset(&conn->in_pktns->crypto.strm) == 0) { + if (conn->in_pktns->crypto.strm.rx.rob && + ngtcp2_rob_data_buffered(conn->in_pktns->crypto.strm.rx.rob)) { /* Address has been validated with token */ if (conn->local.settings.token.len) { return 0; @@ -7386,6 +7631,8 @@ static int conn_read_handshake(ngtcp2_conn *conn, const ngtcp2_path *path, return rv; } + conn->pktns.rtb.persistent_congestion_start_ts = ts; + /* Re-arm loss detection timer here after handshake has been confirmed. */ ngtcp2_conn_set_loss_detection_timer(conn, ts); @@ -7482,8 +7729,8 @@ static size_t conn_server_hs_tx_left(ngtcp2_conn *conn) { static ngtcp2_ssize conn_retransmit_retry_early(ngtcp2_conn *conn, uint8_t *dest, size_t destlen, ngtcp2_tstamp ts) { - return conn_write_pkt(conn, dest, destlen, NULL, NGTCP2_PKT_0RTT, NULL, 0, - NULL, 0, NGTCP2_WRITE_PKT_FLAG_NONE, ts); + return conn_write_pkt(conn, dest, destlen, NULL, NGTCP2_PKT_0RTT, + NGTCP2_WRITE_PKT_FLAG_NONE, ts); } /* @@ -7628,6 +7875,11 @@ static ngtcp2_ssize conn_write_handshake(ngtcp2_conn *conn, uint8_t *dest, paddr = &conn->remote.transport_params.preferred_address; dcid = ngtcp2_ringbuf_push_back(&conn->dcid.unused); ngtcp2_dcid_init(dcid, 1, &paddr->cid, paddr->stateless_reset_token); + + rv = ngtcp2_gaptr_push(&conn->dcid.seqgap, 1, 1); + if (rv != 0) { + return (ngtcp2_ssize)rv; + } } if (conn->remote.transport_params.stateless_reset_token_present) { @@ -7649,13 +7901,6 @@ static ngtcp2_ssize conn_write_handshake(ngtcp2_conn *conn, uint8_t *dest, return (ngtcp2_ssize)rv; } - if (conn->remote.transport_params.preferred_address_present) { - rv = conn_select_preferred_addr(conn); - if (rv != 0) { - return rv; - } - } - cstat->bytes_sent += (size_t)res; return res; @@ -7728,14 +7973,15 @@ static ngtcp2_ssize conn_write_handshake(ngtcp2_conn *conn, uint8_t *dest, * `conn_client_write_handshake` writes client side handshake data and * 0RTT packet. * - * |stream_id|, |fin|, |datav|, and |datavcnt| are stream identifier - * to which 0-RTT data is sent, whether it is a last data chunk in - * this stream, a vector of 0-RTT data, and its number of elements - * respectively. If there is no 0RTT data to send, pass negative - * integer to |stream_id|. The amount of 0RTT data sent is assigned - * to |*pdatalen|. If no data is sent, -1 is assigned. Note that 0 - * length STREAM frame is allowed in QUIC, so 0 might be assigned to - * |*pdatalen|. + * In order to send STREAM data in 0RTT packet, specify + * |vmsg|->stream. |vmsg|->stream.strm, |vmsg|->stream.fin, + * |vmsg|->stream.data, and |vmsg|->stream.datacnt are stream to which + * 0-RTT data is sent, whether it is a last data chunk in this stream, + * a vector of 0-RTT data, and its number of elements respectively. + * The amount of 0RTT data sent is assigned to + * *|vmsg|->stream.pdatalen. If no data is sent, -1 is assigned. + * Note that 0 length STREAM frame is allowed in QUIC, so 0 might be + * assigned to *|vmsg|->stream.pdatalen. * * This function returns 0 if it cannot write any frame because buffer * is too small, or packet is congestion limited. Application should @@ -7745,16 +7991,14 @@ static ngtcp2_ssize conn_write_handshake(ngtcp2_conn *conn, uint8_t *dest, * pointed by |dest| if it succeeds, or one of the following negative * error codes: (TBD). */ -static ngtcp2_ssize -conn_client_write_handshake(ngtcp2_conn *conn, uint8_t *dest, size_t destlen, - ngtcp2_ssize *pdatalen, uint32_t flags, - int64_t stream_id, int fin, const ngtcp2_vec *datav, - size_t datavcnt, ngtcp2_tstamp ts) { - ngtcp2_strm *strm = NULL; +static ngtcp2_ssize conn_client_write_handshake(ngtcp2_conn *conn, + uint8_t *dest, size_t destlen, + ngtcp2_vmsg *vmsg, + ngtcp2_tstamp ts) { int send_stream = 0; ngtcp2_ssize spktlen, early_spktlen; int was_client_initial; - size_t datalen = ngtcp2_vec_len(datav, datavcnt); + size_t datalen; size_t early_datalen = 0; uint8_t wflags = NGTCP2_WRITE_PKT_FLAG_NONE; int ppe_pending = (conn->flags & NGTCP2_CONN_FLAG_PPE_PENDING) != 0; @@ -7763,35 +8007,26 @@ conn_client_write_handshake(ngtcp2_conn *conn, uint8_t *dest, size_t destlen, /* conn->early.ckm might be created in the first call of conn_handshake(). Check it later. */ - if (stream_id != -1 && + if (vmsg && vmsg->type == NGTCP2_VMSG_TYPE_STREAM && !(conn->flags & NGTCP2_CONN_FLAG_EARLY_DATA_REJECTED)) { - strm = ngtcp2_conn_find_stream(conn, stream_id); - if (strm == NULL) { - return NGTCP2_ERR_STREAM_NOT_FOUND; - } - - if (strm->flags & NGTCP2_STRM_FLAG_SHUT_WR) { - return NGTCP2_ERR_STREAM_SHUT_WR; - } - - send_stream = conn_retry_early_payloadlen(conn) == 0 && - /* 0 length STREAM frame is allowed */ - (datalen == 0 || - (datalen > 0 && (strm->tx.max_offset - strm->tx.offset) && - (conn->tx.max_offset - conn->tx.offset))); + datalen = ngtcp2_vec_len(vmsg->stream.data, vmsg->stream.datacnt); + send_stream = + conn_retry_early_payloadlen(conn) == 0 && + /* 0 length STREAM frame is allowed */ + (datalen == 0 || + (datalen > 0 && + (vmsg->stream.strm->tx.max_offset - vmsg->stream.strm->tx.offset) && + (conn->tx.max_offset - conn->tx.offset))); if (send_stream) { - early_datalen = (size_t)ngtcp2_min((uint64_t)datalen, - strm->tx.max_offset - strm->tx.offset); early_datalen = - (size_t)ngtcp2_min((uint64_t)early_datalen, - conn->tx.max_offset - conn->tx.offset) + + conn_enforce_flow_control(conn, vmsg->stream.strm, datalen) + NGTCP2_STREAM_OVERHEAD; - if (flags & NGTCP2_WRITE_STREAM_FLAG_MORE) { - wflags |= NGTCP2_WRITE_PKT_FLAG_STREAM_MORE; + if (vmsg->stream.flags & NGTCP2_WRITE_STREAM_FLAG_MORE) { + wflags |= NGTCP2_WRITE_PKT_FLAG_MORE; } } else { - strm = NULL; + vmsg = NULL; } } @@ -7828,14 +8063,14 @@ conn_client_write_handshake(ngtcp2_conn *conn, uint8_t *dest, size_t destlen, return spktlen; } - early_spktlen = conn_write_pkt(conn, dest, destlen, pdatalen, NGTCP2_PKT_0RTT, - strm, fin, datav, datavcnt, wflags, ts); + early_spktlen = + conn_write_pkt(conn, dest, destlen, vmsg, NGTCP2_PKT_0RTT, wflags, ts); if (early_spktlen < 0) { switch (early_spktlen) { case NGTCP2_ERR_STREAM_DATA_BLOCKED: return spktlen; - case NGTCP2_ERR_WRITE_STREAM_MORE: + case NGTCP2_ERR_WRITE_MORE: conn->pkt.was_client_initial = was_client_initial; conn->pkt.hs_spktlen = spktlen; break; @@ -7928,155 +8163,162 @@ size_t ngtcp2_conn_get_aead_overhead(ngtcp2_conn *conn) { return conn->crypto.aead_overhead; } -int ngtcp2_conn_install_initial_key(ngtcp2_conn *conn, const uint8_t *rx_key, - const uint8_t *rx_iv, - const uint8_t *rx_hp_key, - const uint8_t *tx_key, const uint8_t *tx_iv, - const uint8_t *tx_hp_key, size_t keylen, - size_t ivlen) { +int ngtcp2_conn_install_initial_key( + ngtcp2_conn *conn, const ngtcp2_crypto_aead_ctx *rx_aead_ctx, + const uint8_t *rx_iv, const ngtcp2_crypto_cipher_ctx *rx_hp_ctx, + const ngtcp2_crypto_aead_ctx *tx_aead_ctx, const uint8_t *tx_iv, + const ngtcp2_crypto_cipher_ctx *tx_hp_ctx, size_t ivlen) { ngtcp2_pktns *pktns = conn->in_pktns; int rv; assert(pktns); - if (pktns->crypto.rx.hp_key) { - ngtcp2_vec_del(pktns->crypto.rx.hp_key, conn->mem); - pktns->crypto.rx.hp_key = NULL; - } + conn_call_delete_crypto_cipher_ctx(conn, &pktns->crypto.rx.hp_ctx); + pktns->crypto.rx.hp_ctx.native_handle = NULL; + if (pktns->crypto.rx.ckm) { + conn_call_delete_crypto_aead_ctx(conn, &pktns->crypto.rx.ckm->aead_ctx); ngtcp2_crypto_km_del(pktns->crypto.rx.ckm, conn->mem); pktns->crypto.rx.ckm = NULL; } - if (pktns->crypto.tx.hp_key) { - ngtcp2_vec_del(pktns->crypto.tx.hp_key, conn->mem); - pktns->crypto.tx.hp_key = NULL; - } + + conn_call_delete_crypto_cipher_ctx(conn, &pktns->crypto.tx.hp_ctx); + pktns->crypto.tx.hp_ctx.native_handle = NULL; + if (pktns->crypto.tx.ckm) { + conn_call_delete_crypto_aead_ctx(conn, &pktns->crypto.tx.ckm->aead_ctx); ngtcp2_crypto_km_del(pktns->crypto.tx.ckm, conn->mem); pktns->crypto.tx.ckm = NULL; } - rv = ngtcp2_crypto_km_new(&pktns->crypto.rx.ckm, NULL, 0, rx_key, keylen, - rx_iv, ivlen, conn->mem); + rv = ngtcp2_crypto_km_new(&pktns->crypto.rx.ckm, NULL, 0, NULL, rx_iv, ivlen, + conn->mem); if (rv != 0) { return rv; } - rv = ngtcp2_vec_new(&pktns->crypto.rx.hp_key, rx_hp_key, keylen, conn->mem); + rv = ngtcp2_crypto_km_new(&pktns->crypto.tx.ckm, NULL, 0, NULL, tx_iv, ivlen, + conn->mem); if (rv != 0) { return rv; } - rv = ngtcp2_crypto_km_new(&pktns->crypto.tx.ckm, NULL, 0, tx_key, keylen, - tx_iv, ivlen, conn->mem); - if (rv != 0) { - return rv; - } + /* Take owner ship after we are sure that no failure occurs, so that + caller can delete these contexts on failure. */ + pktns->crypto.rx.ckm->aead_ctx = *rx_aead_ctx; + pktns->crypto.rx.hp_ctx = *rx_hp_ctx; + pktns->crypto.tx.ckm->aead_ctx = *tx_aead_ctx; + pktns->crypto.tx.hp_ctx = *tx_hp_ctx; - return ngtcp2_vec_new(&pktns->crypto.tx.hp_key, tx_hp_key, keylen, conn->mem); + return 0; } -int ngtcp2_conn_install_rx_handshake_key(ngtcp2_conn *conn, const uint8_t *key, - const uint8_t *iv, - const uint8_t *hp_key, size_t keylen, - size_t ivlen) { +int ngtcp2_conn_install_rx_handshake_key( + ngtcp2_conn *conn, const ngtcp2_crypto_aead_ctx *aead_ctx, + const uint8_t *iv, size_t ivlen, const ngtcp2_crypto_cipher_ctx *hp_ctx) { ngtcp2_pktns *pktns = conn->hs_pktns; int rv; assert(pktns); - assert(!pktns->crypto.rx.hp_key); + assert(!pktns->crypto.rx.hp_ctx.native_handle); assert(!pktns->crypto.rx.ckm); - rv = ngtcp2_crypto_km_new(&pktns->crypto.rx.ckm, NULL, 0, key, keylen, iv, - ivlen, conn->mem); + rv = ngtcp2_crypto_km_new(&pktns->crypto.rx.ckm, NULL, 0, aead_ctx, iv, ivlen, + conn->mem); if (rv != 0) { return rv; } - return ngtcp2_vec_new(&pktns->crypto.rx.hp_key, hp_key, keylen, conn->mem); + pktns->crypto.rx.hp_ctx = *hp_ctx; + + return 0; } -int ngtcp2_conn_install_tx_handshake_key(ngtcp2_conn *conn, const uint8_t *key, - const uint8_t *iv, - const uint8_t *hp_key, size_t keylen, - size_t ivlen) { +int ngtcp2_conn_install_tx_handshake_key( + ngtcp2_conn *conn, const ngtcp2_crypto_aead_ctx *aead_ctx, + const uint8_t *iv, size_t ivlen, const ngtcp2_crypto_cipher_ctx *hp_ctx) { ngtcp2_pktns *pktns = conn->hs_pktns; int rv; assert(pktns); - assert(!pktns->crypto.tx.hp_key); + assert(!pktns->crypto.tx.hp_ctx.native_handle); assert(!pktns->crypto.tx.ckm); - rv = ngtcp2_crypto_km_new(&pktns->crypto.tx.ckm, NULL, 0, key, keylen, iv, - ivlen, conn->mem); + rv = ngtcp2_crypto_km_new(&pktns->crypto.tx.ckm, NULL, 0, aead_ctx, iv, ivlen, + conn->mem); if (rv != 0) { return rv; } - return ngtcp2_vec_new(&pktns->crypto.tx.hp_key, hp_key, keylen, conn->mem); + pktns->crypto.tx.hp_ctx = *hp_ctx; + + if (conn->server) { + return ngtcp2_conn_commit_local_transport_params(conn); + } + + return 0; } -int ngtcp2_conn_install_early_key(ngtcp2_conn *conn, const uint8_t *key, - const uint8_t *iv, const uint8_t *hp_key, - size_t keylen, size_t ivlen) { +int ngtcp2_conn_install_early_key(ngtcp2_conn *conn, + const ngtcp2_crypto_aead_ctx *aead_ctx, + const uint8_t *iv, size_t ivlen, + const ngtcp2_crypto_cipher_ctx *hp_ctx) { int rv; - assert(!conn->early.hp_key); + assert(!conn->early.hp_ctx.native_handle); assert(!conn->early.ckm); - rv = ngtcp2_crypto_km_new(&conn->early.ckm, NULL, 0, key, keylen, iv, ivlen, + rv = ngtcp2_crypto_km_new(&conn->early.ckm, NULL, 0, aead_ctx, iv, ivlen, conn->mem); if (rv != 0) { return rv; } - return ngtcp2_vec_new(&conn->early.hp_key, hp_key, keylen, conn->mem); + conn->early.hp_ctx = *hp_ctx; + + return 0; } int ngtcp2_conn_install_rx_key(ngtcp2_conn *conn, const uint8_t *secret, - const uint8_t *key, const uint8_t *iv, - const uint8_t *hp_key, size_t secretlen, - size_t keylen, size_t ivlen) { + size_t secretlen, + const ngtcp2_crypto_aead_ctx *aead_ctx, + const uint8_t *iv, size_t ivlen, + const ngtcp2_crypto_cipher_ctx *hp_ctx) { ngtcp2_pktns *pktns = &conn->pktns; int rv; - assert(!pktns->crypto.rx.hp_key); + assert(!pktns->crypto.rx.hp_ctx.native_handle); assert(!pktns->crypto.rx.ckm); - rv = ngtcp2_crypto_km_new(&pktns->crypto.rx.ckm, secret, secretlen, key, - keylen, iv, ivlen, conn->mem); + rv = ngtcp2_crypto_km_new(&pktns->crypto.rx.ckm, secret, secretlen, aead_ctx, + iv, ivlen, conn->mem); if (rv != 0) { return rv; } - rv = ngtcp2_vec_new(&pktns->crypto.rx.hp_key, hp_key, keylen, conn->mem); - if (rv != 0) { - return rv; - } + pktns->crypto.rx.hp_ctx = *hp_ctx; return 0; } int ngtcp2_conn_install_tx_key(ngtcp2_conn *conn, const uint8_t *secret, - const uint8_t *key, const uint8_t *iv, - const uint8_t *hp_key, size_t secretlen, - size_t keylen, size_t ivlen) { + size_t secretlen, + const ngtcp2_crypto_aead_ctx *aead_ctx, + const uint8_t *iv, size_t ivlen, + const ngtcp2_crypto_cipher_ctx *hp_ctx) { ngtcp2_pktns *pktns = &conn->pktns; int rv; - assert(!pktns->crypto.tx.hp_key); + assert(!pktns->crypto.tx.hp_ctx.native_handle); assert(!pktns->crypto.tx.ckm); - rv = ngtcp2_crypto_km_new(&pktns->crypto.tx.ckm, secret, secretlen, key, - keylen, iv, ivlen, conn->mem); + rv = ngtcp2_crypto_km_new(&pktns->crypto.tx.ckm, secret, secretlen, aead_ctx, + iv, ivlen, conn->mem); if (rv != 0) { return rv; } - rv = ngtcp2_vec_new(&pktns->crypto.tx.hp_key, hp_key, keylen, conn->mem); - if (rv != 0) { - return rv; - } + pktns->crypto.tx.hp_ctx = *hp_ctx; conn->remote.transport_params = conn->remote.pending_transport_params; conn_sync_stream_id_limit(conn); @@ -8150,8 +8392,10 @@ ngtcp2_tstamp ngtcp2_conn_get_expiry(ngtcp2_conn *conn) { ngtcp2_tstamp t1 = ngtcp2_conn_loss_detection_expiry(conn); ngtcp2_tstamp t2 = ngtcp2_conn_ack_delay_expiry(conn); ngtcp2_tstamp t3 = ngtcp2_conn_internal_expiry(conn); + ngtcp2_tstamp t4 = ngtcp2_conn_lost_pkt_expiry(conn); ngtcp2_tstamp res = ngtcp2_min(t1, t2); - return ngtcp2_min(res, t3); + res = ngtcp2_min(res, t3); + return ngtcp2_min(res, t4); } int ngtcp2_conn_handle_expiry(ngtcp2_conn *conn, ngtcp2_tstamp ts) { @@ -8159,6 +8403,8 @@ int ngtcp2_conn_handle_expiry(ngtcp2_conn *conn, ngtcp2_tstamp ts) { ngtcp2_conn_cancel_expired_ack_delay_timer(conn, ts); + ngtcp2_conn_remove_lost_pkt(conn, ts); + if (ngtcp2_conn_loss_detection_expiry(conn) <= ts) { rv = ngtcp2_conn_on_loss_detection_timer(conn, ts); if (rv != 0) { @@ -8188,6 +8434,49 @@ void ngtcp2_conn_cancel_expired_ack_delay_timer(ngtcp2_conn *conn, acktr_cancel_expired_ack_delay_timer(&conn->pktns.acktr, ts); } +ngtcp2_tstamp ngtcp2_conn_lost_pkt_expiry(ngtcp2_conn *conn) { + ngtcp2_tstamp res = UINT64_MAX, ts; + + if (conn->in_pktns) { + ts = ngtcp2_rtb_lost_pkt_ts(&conn->in_pktns->rtb); + if (ts != UINT64_MAX) { + ts += conn_compute_pto(conn, conn->in_pktns); + res = ngtcp2_min(res, ts); + } + } + + if (conn->hs_pktns) { + ts = ngtcp2_rtb_lost_pkt_ts(&conn->hs_pktns->rtb); + if (ts != UINT64_MAX) { + ts += conn_compute_pto(conn, conn->hs_pktns); + res = ngtcp2_min(res, ts); + } + } + + ts = ngtcp2_rtb_lost_pkt_ts(&conn->pktns.rtb); + if (ts != UINT64_MAX) { + ts += conn_compute_pto(conn, &conn->pktns); + res = ngtcp2_min(res, ts); + } + + return res; +} + +void ngtcp2_conn_remove_lost_pkt(ngtcp2_conn *conn, ngtcp2_tstamp ts) { + ngtcp2_tstamp pto; + + if (conn->in_pktns) { + pto = conn_compute_pto(conn, conn->in_pktns); + ngtcp2_rtb_remove_expired_lost_pkt(&conn->in_pktns->rtb, pto, ts); + } + if (conn->hs_pktns) { + pto = conn_compute_pto(conn, conn->hs_pktns); + ngtcp2_rtb_remove_expired_lost_pkt(&conn->hs_pktns->rtb, pto, ts); + } + pto = conn_compute_pto(conn, &conn->pktns); + ngtcp2_rtb_remove_expired_lost_pkt(&conn->pktns.rtb, pto, ts); +} + /* * conn_client_validate_transport_params validates |params| as client. * |params| must be sent with Encrypted Extensions. @@ -8262,7 +8551,7 @@ int ngtcp2_conn_set_remote_transport_params( params); ngtcp2_qlog_parameters_set_transport_params(&conn->qlog, params, conn->server, - /* local = */ 0); + NGTCP2_QLOG_SIDE_REMOTE); if (conn->pktns.crypto.tx.ckm) { conn->remote.transport_params = *params; @@ -8308,20 +8597,86 @@ void ngtcp2_conn_set_early_remote_transport_params( conn->tx.max_offset = p->initial_max_data; ngtcp2_qlog_parameters_set_transport_params(&conn->qlog, p, conn->server, - /* local = */ 0); + NGTCP2_QLOG_SIDE_REMOTE); +} + +int ngtcp2_conn_set_local_transport_params( + ngtcp2_conn *conn, const ngtcp2_transport_params *params) { + assert(conn->server); + assert(params->active_connection_id_limit <= NGTCP2_MAX_DCID_POOL_SIZE); + + if (conn->hs_pktns == NULL || conn->hs_pktns->crypto.tx.ckm) { + return NGTCP2_ERR_INVALID_STATE; + } + + conn->local.settings.transport_params = *params; + + return 0; +} + +int ngtcp2_conn_commit_local_transport_params(ngtcp2_conn *conn) { + const ngtcp2_mem *mem = conn->mem; + ngtcp2_transport_params *params = &conn->local.settings.transport_params; + ngtcp2_scid *scident; + ngtcp2_ksl_it it; + int rv; + + assert(1 == ngtcp2_ksl_len(&conn->scid.set)); + + if (params->active_connection_id_limit == 0) { + params->active_connection_id_limit = + NGTCP2_DEFAULT_ACTIVE_CONNECTION_ID_LIMIT; + } + + params->initial_scid = conn->oscid; + + if (conn->oscid.datalen == 0) { + params->preferred_address_present = 0; + } + + if (conn->server) { + if (params->stateless_reset_token_present) { + it = ngtcp2_ksl_begin(&conn->scid.set); + scident = ngtcp2_ksl_it_get(&it); + + memcpy(scident->token, params->stateless_reset_token, + NGTCP2_STATELESS_RESET_TOKENLEN); + } + + if (params->preferred_address_present) { + scident = ngtcp2_mem_malloc(mem, sizeof(*scident)); + if (scident == NULL) { + return NGTCP2_ERR_NOMEM; + } + + ngtcp2_scid_init(scident, 1, ¶ms->preferred_address.cid, + params->preferred_address.stateless_reset_token); + + rv = ngtcp2_ksl_insert(&conn->scid.set, NULL, &scident->cid, scident); + if (rv != 0) { + ngtcp2_mem_free(mem, scident); + return rv; + } + + conn->scid.last_seq = 1; + } + } + + conn->rx.unsent_max_offset = conn->rx.max_offset = params->initial_max_data; + conn->remote.bidi.unsent_max_streams = params->initial_max_streams_bidi; + conn->remote.bidi.max_streams = params->initial_max_streams_bidi; + conn->remote.uni.unsent_max_streams = params->initial_max_streams_uni; + conn->remote.uni.max_streams = params->initial_max_streams_uni; + + ngtcp2_qlog_parameters_set_transport_params(&conn->qlog, params, conn->server, + NGTCP2_QLOG_SIDE_LOCAL); + + return 0; } void ngtcp2_conn_get_local_transport_params(ngtcp2_conn *conn, ngtcp2_transport_params *params) { *params = conn->local.settings.transport_params; - - /* If params->retry_scid_present is set by application then it - should also specify original_dcid because the destination - connection ID from client after Retry is not what we want - here. */ - if (conn->server && !params->retry_scid_present) { - params->original_dcid = conn->rcid; - } } int ngtcp2_conn_open_bidi_stream(ngtcp2_conn *conn, int64_t *pstream_id, @@ -8410,7 +8765,41 @@ ngtcp2_ssize ngtcp2_conn_writev_stream(ngtcp2_conn *conn, ngtcp2_path *path, int64_t stream_id, const ngtcp2_vec *datav, size_t datavcnt, ngtcp2_tstamp ts) { - ngtcp2_strm *strm = NULL; + ngtcp2_vmsg vmsg, *pvmsg; + ngtcp2_strm *strm; + + if (pdatalen) { + *pdatalen = -1; + } + + if (stream_id != -1) { + strm = ngtcp2_conn_find_stream(conn, stream_id); + if (strm == NULL) { + return NGTCP2_ERR_STREAM_NOT_FOUND; + } + + if (strm->flags & NGTCP2_STRM_FLAG_SHUT_WR) { + return NGTCP2_ERR_STREAM_SHUT_WR; + } + + vmsg.type = NGTCP2_VMSG_TYPE_STREAM; + vmsg.stream.strm = strm; + vmsg.stream.flags = flags; + vmsg.stream.data = datav; + vmsg.stream.datacnt = datavcnt; + vmsg.stream.pdatalen = pdatalen; + + pvmsg = &vmsg; + } else { + pvmsg = NULL; + } + + return ngtcp2_conn_write_vmsg(conn, path, dest, destlen, pvmsg, ts); +} + +ngtcp2_ssize ngtcp2_conn_write_vmsg(ngtcp2_conn *conn, ngtcp2_path *path, + uint8_t *dest, size_t destlen, + ngtcp2_vmsg *vmsg, ngtcp2_tstamp ts) { ngtcp2_ssize nwrite; ngtcp2_pktns *pktns = &conn->pktns; size_t origlen = destlen; @@ -8419,15 +8808,10 @@ ngtcp2_ssize ngtcp2_conn_writev_stream(ngtcp2_conn *conn, ngtcp2_path *path, int ppe_pending = (conn->flags & NGTCP2_CONN_FLAG_PPE_PENDING) != 0; ngtcp2_ssize res = 0; size_t server_hs_tx_left; - int fin = flags & NGTCP2_WRITE_STREAM_FLAG_FIN; conn->log.last_ts = ts; conn->qlog.last_ts = ts; - if (pdatalen) { - *pdatalen = -1; - } - if (path) { ngtcp2_path_copy(path, &conn->dcid.current.ps.path); } @@ -8436,8 +8820,7 @@ ngtcp2_ssize ngtcp2_conn_writev_stream(ngtcp2_conn *conn, ngtcp2_path *path, case NGTCP2_CS_CLIENT_INITIAL: case NGTCP2_CS_CLIENT_WAIT_HANDSHAKE: case NGTCP2_CS_CLIENT_TLS_HANDSHAKE_FAILED: - nwrite = conn_client_write_handshake(conn, dest, destlen, pdatalen, flags, - stream_id, fin, datav, datavcnt, ts); + nwrite = conn_client_write_handshake(conn, dest, destlen, vmsg, ts); if (nwrite < 0 || conn->state != NGTCP2_CS_POST_HANDSHAKE) { return nwrite; } @@ -8494,18 +8877,15 @@ ngtcp2_ssize ngtcp2_conn_writev_stream(ngtcp2_conn *conn, ngtcp2_path *path, return rv; } - if (stream_id != -1) { - strm = ngtcp2_conn_find_stream(conn, stream_id); - if (strm == NULL) { - return NGTCP2_ERR_STREAM_NOT_FOUND; - } - - if (strm->flags & NGTCP2_STRM_FLAG_SHUT_WR) { - return NGTCP2_ERR_STREAM_SHUT_WR; - } - - if (flags & NGTCP2_WRITE_STREAM_FLAG_MORE) { - wflags |= NGTCP2_WRITE_PKT_FLAG_STREAM_MORE; + if (vmsg) { + switch (vmsg->type) { + case NGTCP2_VMSG_TYPE_STREAM: + if (vmsg->stream.flags & NGTCP2_WRITE_STREAM_FLAG_MORE) { + wflags |= NGTCP2_WRITE_PKT_FLAG_MORE; + } + break; + default: + break; } } @@ -8514,8 +8894,8 @@ ngtcp2_ssize ngtcp2_conn_writev_stream(ngtcp2_conn *conn, ngtcp2_path *path, conn->pkt.hs_spktlen = 0; /* dest and destlen have already been adjusted in ppe in the first run. They are adjusted for probe packet later. */ - nwrite = conn_write_pkt(conn, dest, destlen, pdatalen, NGTCP2_PKT_SHORT, - strm, fin, datav, datavcnt, wflags, ts); + nwrite = + conn_write_pkt(conn, dest, destlen, vmsg, NGTCP2_PKT_SHORT, wflags, ts); goto fin; } else { if (conn->state == NGTCP2_CS_POST_HANDSHAKE) { @@ -8557,14 +8937,14 @@ ngtcp2_ssize ngtcp2_conn_writev_stream(ngtcp2_conn *conn, ngtcp2_path *path, "transmit probe pkt left=%zu", conn->pktns.rtb.probe_pkt_left); - nwrite = conn_write_pkt(conn, dest, destlen, pdatalen, NGTCP2_PKT_SHORT, - strm, fin, datav, datavcnt, wflags, ts); + nwrite = + conn_write_pkt(conn, dest, destlen, vmsg, NGTCP2_PKT_SHORT, wflags, ts); goto fin; } - nwrite = conn_write_pkt(conn, dest, destlen, pdatalen, NGTCP2_PKT_SHORT, strm, - fin, datav, datavcnt, wflags, ts); + nwrite = + conn_write_pkt(conn, dest, destlen, vmsg, NGTCP2_PKT_SHORT, wflags, ts); if (nwrite) { assert(nwrite != NGTCP2_ERR_NOBUF); goto fin; @@ -8590,36 +8970,15 @@ ngtcp2_ssize ngtcp2_conn_writev_stream(ngtcp2_conn *conn, ngtcp2_path *path, return nwrite; } -ngtcp2_ssize ngtcp2_conn_write_connection_close(ngtcp2_conn *conn, - ngtcp2_path *path, +static ngtcp2_ssize conn_write_connection_close(ngtcp2_conn *conn, uint8_t *dest, size_t destlen, + uint8_t pkt_type, uint64_t error_code, ngtcp2_tstamp ts) { ngtcp2_pktns *in_pktns = conn->in_pktns; ngtcp2_pktns *hs_pktns = conn->hs_pktns; ngtcp2_ssize res = 0, nwrite; ngtcp2_frame fr; - uint8_t pkt_type; - - conn->log.last_ts = ts; - conn->qlog.last_ts = ts; - - if (conn_check_pkt_num_exhausted(conn)) { - return NGTCP2_ERR_PKT_NUM_EXHAUSTED; - } - - switch (conn->state) { - case NGTCP2_CS_CLIENT_INITIAL: - case NGTCP2_CS_CLOSING: - case NGTCP2_CS_DRAINING: - return NGTCP2_ERR_INVALID_STATE; - default: - break; - } - - if (path) { - ngtcp2_path_copy(path, &conn->dcid.current.ps.path); - } fr.type = NGTCP2_FRAME_CONNECTION_CLOSE; fr.connection_close.error_code = error_code; @@ -8627,18 +8986,6 @@ ngtcp2_ssize ngtcp2_conn_write_connection_close(ngtcp2_conn *conn, fr.connection_close.reasonlen = 0; fr.connection_close.reason = NULL; - if (conn->state == NGTCP2_CS_POST_HANDSHAKE) { - pkt_type = NGTCP2_PKT_SHORT; - } else if (hs_pktns && hs_pktns->crypto.tx.ckm) { - pkt_type = NGTCP2_PKT_HANDSHAKE; - } else if (in_pktns && in_pktns->crypto.tx.ckm) { - pkt_type = NGTCP2_PKT_INITIAL; - } else { - /* This branch is taken if server has not read any Initial packet - from client. */ - return NGTCP2_ERR_INVALID_STATE; - } - if (!(conn->flags & NGTCP2_CONN_FLAG_HANDSHAKE_CONFIRMED) && pkt_type != NGTCP2_PKT_INITIAL) { if (in_pktns && conn->server) { @@ -8683,9 +9030,61 @@ ngtcp2_ssize ngtcp2_conn_write_connection_close(ngtcp2_conn *conn, return NGTCP2_ERR_NOBUF; } + return res; +} + +ngtcp2_ssize ngtcp2_conn_write_connection_close(ngtcp2_conn *conn, + ngtcp2_path *path, + uint8_t *dest, size_t destlen, + uint64_t error_code, + ngtcp2_tstamp ts) { + ngtcp2_pktns *in_pktns = conn->in_pktns; + ngtcp2_pktns *hs_pktns = conn->hs_pktns; + uint8_t pkt_type; + ngtcp2_ssize nwrite; + + conn->log.last_ts = ts; + conn->qlog.last_ts = ts; + + if (conn_check_pkt_num_exhausted(conn)) { + return NGTCP2_ERR_PKT_NUM_EXHAUSTED; + } + + switch (conn->state) { + case NGTCP2_CS_CLIENT_INITIAL: + case NGTCP2_CS_CLOSING: + case NGTCP2_CS_DRAINING: + return NGTCP2_ERR_INVALID_STATE; + default: + break; + } + + if (path) { + ngtcp2_path_copy(path, &conn->dcid.current.ps.path); + } + + if (conn->state == NGTCP2_CS_POST_HANDSHAKE || + (conn->server && conn->pktns.crypto.tx.ckm)) { + pkt_type = NGTCP2_PKT_SHORT; + } else if (hs_pktns && hs_pktns->crypto.tx.ckm) { + pkt_type = NGTCP2_PKT_HANDSHAKE; + } else if (in_pktns && in_pktns->crypto.tx.ckm) { + pkt_type = NGTCP2_PKT_INITIAL; + } else { + /* This branch is taken if server has not read any Initial packet + from client. */ + return NGTCP2_ERR_INVALID_STATE; + } + + nwrite = conn_write_connection_close(conn, dest, destlen, pkt_type, + error_code, ts); + if (nwrite < 0) { + return nwrite; + } + conn->state = NGTCP2_CS_CLOSING; - return res; + return nwrite; } ngtcp2_ssize ngtcp2_conn_write_application_close(ngtcp2_conn *conn, @@ -8694,6 +9093,7 @@ ngtcp2_ssize ngtcp2_conn_write_application_close(ngtcp2_conn *conn, uint64_t app_error_code, ngtcp2_tstamp ts) { ngtcp2_ssize nwrite; + ngtcp2_ssize res = 0; ngtcp2_frame fr; conn->log.last_ts = ts; @@ -8704,12 +9104,38 @@ ngtcp2_ssize ngtcp2_conn_write_application_close(ngtcp2_conn *conn, } switch (conn->state) { - case NGTCP2_CS_POST_HANDSHAKE: - break; - default: + case NGTCP2_CS_CLIENT_INITIAL: + case NGTCP2_CS_CLOSING: + case NGTCP2_CS_DRAINING: return NGTCP2_ERR_INVALID_STATE; + default: + break; } + if (!(conn->flags & NGTCP2_CONN_FLAG_HANDSHAKE_CONFIRMED)) { + nwrite = conn_write_connection_close(conn, dest, destlen, + conn->hs_pktns->crypto.tx.ckm + ? NGTCP2_PKT_HANDSHAKE + : NGTCP2_PKT_INITIAL, + NGTCP2_APPLICATION_ERROR, ts); + if (nwrite < 0) { + return nwrite; + } + res = nwrite; + dest += nwrite; + destlen -= (size_t)nwrite; + } + + if (conn->state != NGTCP2_CS_POST_HANDSHAKE) { + assert(res); + + if (!conn->server || !conn->pktns.crypto.tx.ckm) { + return res; + } + } + + assert(conn->pktns.crypto.tx.ckm); + if (path) { ngtcp2_path_copy(path, &conn->dcid.current.ps.path); } @@ -8728,13 +9154,15 @@ ngtcp2_ssize ngtcp2_conn_write_application_close(ngtcp2_conn *conn, return nwrite; } - if (nwrite == 0) { + res += nwrite; + + if (res == 0) { return NGTCP2_ERR_NOBUF; } conn->state = NGTCP2_CS_CLOSING; - return nwrite; + return res; } int ngtcp2_conn_is_in_closing_period(ngtcp2_conn *conn) { @@ -8780,7 +9208,7 @@ int ngtcp2_conn_close_stream_if_shut_rdwr(ngtcp2_conn *conn, ngtcp2_strm *strm, if ((strm->flags & NGTCP2_STRM_FLAG_SHUT_RDWR) == NGTCP2_STRM_FLAG_SHUT_RDWR && ((strm->flags & NGTCP2_STRM_FLAG_RECV_RST) || - ngtcp2_rob_first_gap_offset(&strm->rx.rob) == strm->rx.last_offset) && + ngtcp2_strm_rx_offset(strm) == strm->rx.last_offset) && (((strm->flags & NGTCP2_STRM_FLAG_SENT_RST) && (strm->flags & NGTCP2_STRM_FLAG_RST_ACKED)) || (!(strm->flags & NGTCP2_STRM_FLAG_SENT_RST) && @@ -8974,17 +9402,12 @@ uint32_t ngtcp2_conn_get_negotiated_version(ngtcp2_conn *conn) { int ngtcp2_conn_early_data_rejected(ngtcp2_conn *conn) { ngtcp2_pktns *pktns = &conn->pktns; ngtcp2_rtb *rtb = &conn->pktns.rtb; - ngtcp2_frame_chain *frc = NULL; int rv; conn->flags |= NGTCP2_CONN_FLAG_EARLY_DATA_REJECTED; - ngtcp2_rtb_remove_all(rtb, &frc, &conn->cstat); - - rv = conn_resched_frames(conn, pktns, &frc); + rv = ngtcp2_rtb_remove_all(rtb, conn, pktns, &conn->cstat); if (rv != 0) { - assert(ngtcp2_err_is_fatal(rv)); - ngtcp2_frame_chain_list_del(frc, conn->mem); return rv; } @@ -8997,8 +9420,9 @@ void ngtcp2_conn_update_rtt(ngtcp2_conn *conn, ngtcp2_duration rtt, rtt = ngtcp2_max(rtt, NGTCP2_GRANULARITY); + cstat->latest_rtt = rtt; + if (cstat->min_rtt == UINT64_MAX) { - cstat->latest_rtt = rtt; cstat->min_rtt = rtt; cstat->smoothed_rtt = rtt; cstat->rttvar = rtt / 2; @@ -9014,8 +9438,6 @@ void ngtcp2_conn_update_rtt(ngtcp2_conn *conn, ngtcp2_duration rtt, rtt -= ack_delay; } - cstat->latest_rtt = rtt; - cstat->rttvar = (cstat->rttvar * 3 + (cstat->smoothed_rtt < rtt ? rtt - cstat->smoothed_rtt : cstat->smoothed_rtt - rtt)) / @@ -9047,7 +9469,7 @@ static ngtcp2_pktns *conn_get_earliest_pktns(ngtcp2_conn *conn, ngtcp2_tstamp earliest_ts = times[NGTCP2_PKTNS_ID_INITIAL]; for (i = NGTCP2_PKTNS_ID_HANDSHAKE; i < NGTCP2_PKTNS_ID_MAX; ++i) { - if (ns[i] == NULL || ns[i]->rtb.num_ack_eliciting == 0 || + if (ns[i] == NULL || ns[i]->rtb.num_retransmittable == 0 || (times[i] == UINT64_MAX || (earliest_ts != UINT64_MAX && times[i] >= earliest_ts) || (i == NGTCP2_PKTNS_ID_APP && @@ -9096,9 +9518,9 @@ void ngtcp2_conn_set_loss_detection_timer(ngtcp2_conn *conn, ngtcp2_tstamp ts) { return; } - if ((!in_pktns || in_pktns->rtb.num_ack_eliciting == 0) && - (!hs_pktns || hs_pktns->rtb.num_ack_eliciting == 0) && - (pktns->rtb.num_ack_eliciting == 0 || + if ((!in_pktns || in_pktns->rtb.num_retransmittable == 0) && + (!hs_pktns || hs_pktns->rtb.num_retransmittable == 0) && + (pktns->rtb.num_retransmittable == 0 || !(conn->flags & NGTCP2_CONN_FLAG_HANDSHAKE_COMPLETED)) && (conn->server || (conn->flags & (NGTCP2_CONN_FLAG_SERVER_ADDR_VERIFIED | @@ -9132,36 +9554,6 @@ void ngtcp2_conn_set_loss_detection_timer(ngtcp2_conn *conn, ngtcp2_tstamp ts) { (uint64_t)(timeout / NGTCP2_MILLISECONDS)); } -/* - * conn_on_crypto_timeout is called when handshake packets which - * belong to |pktns| are lost. - * - * This function returns 0 if it succeeds, or one of the following - * negative error codes: - * - * NGTCP2_ERR_NOMEM - * Out of memory. - */ -static int conn_on_crypto_timeout(ngtcp2_conn *conn, ngtcp2_pktns *pktns) { - ngtcp2_frame_chain *frc = NULL; - int rv; - - rv = ngtcp2_rtb_on_crypto_timeout(&pktns->rtb, &frc, &conn->cstat); - if (rv != 0) { - assert(ngtcp2_err_is_fatal(rv)); - ngtcp2_frame_chain_list_del(frc, conn->mem); - return rv; - } - - rv = conn_resched_frames(conn, pktns, &frc); - if (rv != 0) { - ngtcp2_frame_chain_list_del(frc, conn->mem); - return rv; - } - - return 0; -} - int ngtcp2_conn_on_loss_detection_timer(ngtcp2_conn *conn, ngtcp2_tstamp ts) { ngtcp2_conn_stat *cstat = &conn->cstat; int rv; @@ -9203,16 +9595,8 @@ int ngtcp2_conn_on_loss_detection_timer(ngtcp2_conn *conn, ngtcp2_tstamp ts) { if (!conn->server && !(conn->flags & NGTCP2_CONN_FLAG_HANDSHAKE_COMPLETED)) { if (hs_pktns->crypto.tx.ckm) { - rv = conn_on_crypto_timeout(conn, hs_pktns); - if (rv != 0) { - return rv; - } hs_pktns->rtb.probe_pkt_left = 1; } else { - rv = conn_on_crypto_timeout(conn, in_pktns); - if (rv != 0) { - return rv; - } in_pktns->rtb.probe_pkt_left = 1; } } else { @@ -9223,10 +9607,6 @@ int ngtcp2_conn_on_loss_detection_timer(ngtcp2_conn *conn, ngtcp2_tstamp ts) { switch (earliest_pktns->rtb.pktns_id) { case NGTCP2_PKTNS_ID_INITIAL: assert(in_pktns); - rv = conn_on_crypto_timeout(conn, in_pktns); - if (rv != 0) { - return rv; - } in_pktns->rtb.probe_pkt_left = 1; if (!conn->server) { break; @@ -9235,10 +9615,6 @@ int ngtcp2_conn_on_loss_detection_timer(ngtcp2_conn *conn, ngtcp2_tstamp ts) { /* fall through */ case NGTCP2_PKTNS_ID_HANDSHAKE: assert(hs_pktns); - rv = conn_on_crypto_timeout(conn, hs_pktns); - if (rv != 0) { - return rv; - } hs_pktns->rtb.probe_pkt_left = 1; break; case NGTCP2_PKTNS_ID_APP: @@ -9580,8 +9956,12 @@ const ngtcp2_crypto_ctx *ngtcp2_conn_get_initial_crypto_ctx(ngtcp2_conn *conn) { } void ngtcp2_conn_set_retry_aead(ngtcp2_conn *conn, - const ngtcp2_crypto_aead *aead) { + const ngtcp2_crypto_aead *aead, + const ngtcp2_crypto_aead_ctx *aead_ctx) { + assert(!conn->crypto.retry_aead_ctx.native_handle); + conn->crypto.retry_aead = *aead; + conn->crypto.retry_aead_ctx = *aead_ctx; } void ngtcp2_conn_set_crypto_ctx(ngtcp2_conn *conn, @@ -9623,6 +10003,23 @@ int ngtcp2_conn_is_local_stream(ngtcp2_conn *conn, int64_t stream_id) { int ngtcp2_conn_is_server(ngtcp2_conn *conn) { return conn->server; } +int ngtcp2_conn_after_retry(ngtcp2_conn *conn) { + return (conn->flags & NGTCP2_CONN_FLAG_RECV_RETRY) != 0; +} + +int ngtcp2_conn_set_stream_user_data(ngtcp2_conn *conn, int64_t stream_id, + void *stream_user_data) { + ngtcp2_strm *strm = ngtcp2_conn_find_stream(conn, stream_id); + + if (strm == NULL) { + return NGTCP2_ERR_STREAM_NOT_FOUND; + } + + strm->stream_user_data = stream_user_data; + + return 0; +} + void ngtcp2_path_challenge_entry_init(ngtcp2_path_challenge_entry *pcent, const uint8_t *data) { memcpy(pcent->data, data, sizeof(pcent->data)); @@ -9631,6 +10028,7 @@ void ngtcp2_path_challenge_entry_init(ngtcp2_path_challenge_entry *pcent, void ngtcp2_settings_default(ngtcp2_settings *settings) { memset(settings, 0, sizeof(*settings)); settings->cc_algo = NGTCP2_CC_ALGO_CUBIC; + settings->initial_rtt = NGTCP2_DEFAULT_INITIAL_RTT; settings->transport_params.max_udp_payload_size = NGTCP2_DEFAULT_MAX_UDP_PAYLOAD_SIZE; settings->transport_params.ack_delay_exponent = @@ -9647,13 +10045,12 @@ void ngtcp2_settings_default(ngtcp2_settings *settings) { ngtcp2_ssize ngtcp2_pkt_write_connection_close( uint8_t *dest, size_t destlen, const ngtcp2_cid *dcid, const ngtcp2_cid *scid, uint64_t error_code, ngtcp2_encrypt encrypt, - const ngtcp2_crypto_aead *aead, const uint8_t *key, const uint8_t *iv, - ngtcp2_hp_mask hp_mask, const ngtcp2_crypto_cipher *hp, - const uint8_t *hp_key) { + const ngtcp2_crypto_aead *aead, const ngtcp2_crypto_aead_ctx *aead_ctx, + const uint8_t *iv, ngtcp2_hp_mask hp_mask, const ngtcp2_crypto_cipher *hp, + const ngtcp2_crypto_cipher_ctx *hp_ctx) { ngtcp2_pkt_hd hd; ngtcp2_crypto_km ckm; ngtcp2_crypto_cc cc; - ngtcp2_vec hp_key_vec; ngtcp2_ppe ppe; ngtcp2_frame fr = {0}; int rv; @@ -9663,9 +10060,8 @@ ngtcp2_ssize ngtcp2_pkt_write_connection_close( NGTCP2_PROTO_VER, /* len = */ 0); ngtcp2_vec_init(&ckm.secret, NULL, 0); - ngtcp2_vec_init(&ckm.key, key, 16); ngtcp2_vec_init(&ckm.iv, iv, 12); - ngtcp2_vec_init(&hp_key_vec, hp_key, 16); + ckm.aead_ctx = *aead_ctx; ckm.pkt_num = 0; ckm.flags = NGTCP2_CRYPTO_KM_FLAG_NONE; @@ -9673,7 +10069,7 @@ ngtcp2_ssize ngtcp2_pkt_write_connection_close( cc.aead = *aead; cc.hp = *hp; cc.ckm = &ckm; - cc.hp_key = &hp_key_vec; + cc.hp_ctx = *hp_ctx; cc.encrypt = encrypt; cc.hp_mask = hp_mask; diff --git a/deps/ngtcp2/lib/ngtcp2_conn.h b/deps/ngtcp2/lib/ngtcp2_conn.h index 39b5b029385fcb..d7782b1114a53d 100644 --- a/deps/ngtcp2/lib/ngtcp2_conn.h +++ b/deps/ngtcp2/lib/ngtcp2_conn.h @@ -106,7 +106,7 @@ typedef enum { /* NGTCP2_MAX_NON_ACK_TX_PKT is the maximum number of continuous non ACK-eliciting packets. */ -#define NGTCP2_MAX_NON_ACK_TX_PKT 10 +#define NGTCP2_MAX_NON_ACK_TX_PKT 3 /* * ngtcp2_max_frame is defined so that it covers the largest ACK @@ -186,7 +186,7 @@ typedef struct { uint8_t pkt_type; } ngtcp2_crypto_data; -typedef struct { +typedef struct ngtcp2_pktns { struct { /* last_pkt_num is the packet number which the local endpoint sent last time.*/ @@ -203,6 +203,9 @@ typedef struct { ngtcp2_gaptr pngap; /* max_pkt_num is the largest packet number received so far. */ int64_t max_pkt_num; + /* max_pkt_ts is the timestamp when max_pkt_num packet is + received. */ + ngtcp2_tstamp max_pkt_ts; /* * buffed_pkts is buffered packets which cannot be decrypted with * the current encryption level. @@ -236,16 +239,16 @@ typedef struct { /* ckm is a cryptographic key, and iv to encrypt outgoing packets. */ ngtcp2_crypto_km *ckm; - /* hp_key is header protection key. */ - ngtcp2_vec *hp_key; + /* hp_ctx is cipher context for packet header protection. */ + ngtcp2_crypto_cipher_ctx hp_ctx; } tx; struct { /* ckm is a cryptographic key, and iv to decrypt incoming packets. */ ngtcp2_crypto_km *ckm; - /* hp_key is header protection key. */ - ngtcp2_vec *hp_key; + /* hp_ctx is cipher context for packet header protection. */ + ngtcp2_crypto_cipher_ctx hp_ctx; } rx; ngtcp2_strm strm; @@ -286,7 +289,10 @@ struct ngtcp2_conn { /* retired is a set of CID retired by local endpoint. Keep them in 3*PTO to catch packets in flight along the old path. */ ngtcp2_ringbuf retired; - /* retire_prior_to is the larget retire_prior_to received so + /* seqgap tracks received sequence numbers in order to ignore + retransmitted duplicated NEW_CONNECTION_ID frame. */ + ngtcp2_gaptr seqgap; + /* retire_prior_to is the largest retire_prior_to received so far. */ uint64_t retire_prior_to; /* num_retire_queued is the number of RETIRE_CONNECTION_ID frames @@ -352,7 +358,7 @@ struct ngtcp2_conn { struct { ngtcp2_crypto_km *ckm; - ngtcp2_vec *hp_key; + ngtcp2_crypto_cipher_ctx hp_ctx; } early; struct { @@ -432,8 +438,12 @@ struct ngtcp2_conn { size_t aead_overhead; /* decrypt_buf is a buffer which is used to write decrypted data. */ ngtcp2_vec decrypt_buf; - /* retry_aead is AEAD to verify Retry packet integrity. */ + /* retry_aead is AEAD to verify Retry packet integrity. It is + used by client only. */ ngtcp2_crypto_aead retry_aead; + /* retry_aead_ctx is AEAD cipher context to verify Retry packet + integrity. It is used by client only. */ + ngtcp2_crypto_aead_ctx retry_aead_ctx; /* tls_error is TLS related error. */ int tls_error; } crypto; @@ -470,6 +480,33 @@ struct ngtcp2_conn { int server; }; +typedef enum ngtcp2_vmsg_type { + NGTCP2_VMSG_TYPE_STREAM, +} ngtcp2_vmsg_type; + +typedef struct ngtcp2_vmsg_stream { + /* strm is a stream that data is sent to. */ + ngtcp2_strm *strm; + /* flags is bitwise OR of zero or more of + ngtcp2_write_stream_flag. */ + uint32_t flags; + /* data is the pointer to ngtcp2_vec array which contains the stream + data to send. */ + const ngtcp2_vec *data; + /* datacnt is the number of ngtcp2_vec pointed by data. */ + size_t datacnt; + /* *pdatalen is the pointer to the variable which the number of + bytes written is assigned to if pdatalen is not NULL. */ + ngtcp2_ssize *pdatalen; +} ngtcp2_vmsg_stream; + +typedef struct ngtcp2_vmsg { + ngtcp2_vmsg_type type; + union { + ngtcp2_vmsg_stream stream; + }; +} ngtcp2_vmsg; + /* * ngtcp2_conn_sched_ack stores packet number |pkt_num| and its * reception timestamp |ts| in order to send its ACK. @@ -591,6 +628,10 @@ int ngtcp2_conn_tx_strmq_push(ngtcp2_conn *conn, ngtcp2_strm *strm); */ ngtcp2_tstamp ngtcp2_conn_internal_expiry(ngtcp2_conn *conn); +ngtcp2_ssize ngtcp2_conn_write_vmsg(ngtcp2_conn *conn, ngtcp2_path *path, + uint8_t *dest, size_t destlen, + ngtcp2_vmsg *vmsg, ngtcp2_tstamp ts); + /* * ngtcp2_conn_write_single_frame_pkt writes a packet which contains |fr| * frame only in the buffer pointed by |dest| whose length if @@ -612,4 +653,46 @@ ngtcp2_conn_write_single_frame_pkt(ngtcp2_conn *conn, uint8_t *dest, const ngtcp2_cid *dcid, ngtcp2_frame *fr, uint8_t rtb_flags, ngtcp2_tstamp ts); +/* + * ngtcp2_conn_commit_local_transport_params commits the local + * transport parameters, which is currently set to + * conn->local.settings.transport_params. This function will do some + * amends on transport parameters for adjusting default values. + * + * This function returns 0 if it succeeds, or one of the following + * negative error codes: + * + * NGTCP2_ERR_NOMEM + * Out of memory. + * NGTCP2_ERR_INVALID_ARGUMENT + * CID in preferred address equals to the original SCID. + */ +int ngtcp2_conn_commit_local_transport_params(ngtcp2_conn *conn); + +/* + * ngtcp2_conn_lost_pkt_expiry returns the earliest expiry time of + * lost packet. + */ +ngtcp2_tstamp ngtcp2_conn_lost_pkt_expiry(ngtcp2_conn *conn); + +/* + * ngtcp2_conn_remove_lost_pkt removes the expired lost packet. + */ +void ngtcp2_conn_remove_lost_pkt(ngtcp2_conn *conn, ngtcp2_tstamp ts); + +/* + * ngtcp2_conn_resched_frames reschedules frames linked from |*pfrc| + * for retransmission. + * + * This function returns 0 if it succeeds, or one of the following + * negative error codes: + * + * NGTCP2_ERR_NOMEM + * Out of memory. + */ +int ngtcp2_conn_resched_frames(ngtcp2_conn *conn, ngtcp2_pktns *pktns, + ngtcp2_frame_chain **pfrc); + +uint64_t ngtcp2_conn_tx_strmq_first_cycle(ngtcp2_conn *conn); + #endif /* NGTCP2_CONN_H */ diff --git a/deps/ngtcp2/lib/ngtcp2_conv.c b/deps/ngtcp2/lib/ngtcp2_conv.c index 7811c84d0bc19a..18cf314791384d 100644 --- a/deps/ngtcp2/lib/ngtcp2_conv.c +++ b/deps/ngtcp2/lib/ngtcp2_conv.c @@ -45,19 +45,19 @@ uint64_t ngtcp2_get_uint48(const uint8_t *p) { uint32_t ngtcp2_get_uint32(const uint8_t *p) { uint32_t n; memcpy(&n, p, 4); - return ntohl(n); + return ngtcp2_ntohl(n); } uint32_t ngtcp2_get_uint24(const uint8_t *p) { uint32_t n = 0; memcpy(((uint8_t *)&n) + 1, p, 3); - return ntohl(n); + return ngtcp2_ntohl(n); } uint16_t ngtcp2_get_uint16(const uint8_t *p) { uint16_t n; memcpy(&n, p, 2); - return ntohs(n); + return ngtcp2_ntohs(n); } uint64_t ngtcp2_get_varint(size_t *plen, const uint8_t *p) { @@ -76,11 +76,11 @@ uint64_t ngtcp2_get_varint(size_t *plen, const uint8_t *p) { case 2: memcpy(&n, p, 2); n.b[0] &= 0x3f; - return ntohs(n.n16); + return ngtcp2_ntohs(n.n16); case 4: memcpy(&n, p, 4); n.b[0] &= 0x3f; - return ntohl(n.n32); + return ngtcp2_ntohl(n.n32); case 8: memcpy(&n, p, 8); n.b[0] &= 0x3f; @@ -116,17 +116,17 @@ uint8_t *ngtcp2_put_uint48be(uint8_t *p, uint64_t n) { } uint8_t *ngtcp2_put_uint32be(uint8_t *p, uint32_t n) { - n = htonl(n); + n = ngtcp2_htonl(n); return ngtcp2_cpymem(p, (const uint8_t *)&n, sizeof(n)); } uint8_t *ngtcp2_put_uint24be(uint8_t *p, uint32_t n) { - n = htonl(n); + n = ngtcp2_htonl(n); return ngtcp2_cpymem(p, ((const uint8_t *)&n) + 1, 3); } uint8_t *ngtcp2_put_uint16be(uint8_t *p, uint16_t n) { - n = htons(n); + n = ngtcp2_htons(n); return ngtcp2_cpymem(p, (const uint8_t *)&n, sizeof(n)); } diff --git a/deps/ngtcp2/lib/ngtcp2_conv.h b/deps/ngtcp2/lib/ngtcp2_conv.h index f3336f44a92dad..227470b91927b8 100644 --- a/deps/ngtcp2/lib/ngtcp2_conv.h +++ b/deps/ngtcp2/lib/ngtcp2_conv.h @@ -37,6 +37,10 @@ # include #endif /* HAVE_NETINET_IN_H */ +#ifdef HAVE_BYTESWAP_H +# include +#endif /* HAVE_BYTESWAP_H */ + #ifdef HAVE_ENDIAN_H # include #endif /* HAVE_ENDIAN_H */ @@ -47,15 +51,25 @@ #include +#if defined HAVE_BSWAP_64 || HAVE_DECL_BSWAP_64 +# define ngtcp2_bswap64 bswap_64 +#else /* !HAVE_BSWAP_64 */ +# define ngtcp2_bswap64(N) \ + ((uint64_t)(ntohl((uint32_t)(N))) << 32 | ntohl((uint32_t)((N) >> 32))) +#endif /* !HAVE_BSWAP_64 */ + #if defined HAVE_BE64TOH || HAVE_DECL_BE64TOH # define ngtcp2_ntohl64(N) be64toh(N) # define ngtcp2_htonl64(N) htobe64(N) #else /* !HAVE_BE64TOH */ -# define ngtcp2_bswap64(N) \ - ((uint64_t)(ntohl((uint32_t)(N))) << 32 | ntohl((uint32_t)((N) >> 32))) -# define ngtcp2_ntohl64(N) ngtcp2_bswap64(N) -# define ngtcp2_htonl64(N) ngtcp2_bswap64(N) -#endif /* !HAVE_BE64TOH */ +# if defined WORDS_BIGENDIAN +# define ngtcp2_ntohl64(N) (N) +# define ngtcp2_htonl64(N) (N) +# else /* !WORDS_BIGENDIAN */ +# define ngtcp2_ntohl64(N) ngtcp2_bswap64(N) +# define ngtcp2_htonl64(N) ngtcp2_bswap64(N) +# endif /* !WORDS_BIGENDIAN */ +#endif /* !HAVE_BE64TOH */ #if defined(WIN32) /* Windows requires ws2_32 library for ntonl family functions. We @@ -68,7 +82,7 @@ # define STIN static inline # endif -STIN uint32_t htonl(uint32_t hostlong) { +STIN uint32_t ngtcp2_htonl(uint32_t hostlong) { uint32_t res; unsigned char *p = (unsigned char *)&res; *p++ = hostlong >> 24; @@ -78,7 +92,7 @@ STIN uint32_t htonl(uint32_t hostlong) { return res; } -STIN uint16_t htons(uint16_t hostshort) { +STIN uint16_t ngtcp2_htons(uint16_t hostshort) { uint16_t res; unsigned char *p = (unsigned char *)&res; *p++ = hostshort >> 8; @@ -86,7 +100,7 @@ STIN uint16_t htons(uint16_t hostshort) { return res; } -STIN uint32_t ntohl(uint32_t netlong) { +STIN uint32_t ngtcp2_ntohl(uint32_t netlong) { uint32_t res; unsigned char *p = (unsigned char *)&netlong; res = *p++ << 24; @@ -96,7 +110,7 @@ STIN uint32_t ntohl(uint32_t netlong) { return res; } -STIN uint16_t ntohs(uint16_t netshort) { +STIN uint16_t ngtcp2_ntohs(uint16_t netshort) { uint16_t res; unsigned char *p = (unsigned char *)&netshort; res = *p++ << 8; @@ -104,7 +118,14 @@ STIN uint16_t ntohs(uint16_t netshort) { return res; } -#endif /* WIN32 */ +#else /* !WIN32 */ + +# define ngtcp2_htonl htonl +# define ngtcp2_htons htons +# define ngtcp2_ntohl ntohl +# define ngtcp2_ntohs ntohs + +#endif /* !WIN32 */ /* * ngtcp2_get_uint64 reads 8 bytes from |p| as 64 bits unsigned diff --git a/deps/ngtcp2/lib/ngtcp2_crypto.c b/deps/ngtcp2/lib/ngtcp2_crypto.c index 26eacb87299336..5cfe145ec9b6a8 100644 --- a/deps/ngtcp2/lib/ngtcp2_crypto.c +++ b/deps/ngtcp2/lib/ngtcp2_crypto.c @@ -32,10 +32,11 @@ #include "ngtcp2_conn.h" int ngtcp2_crypto_km_new(ngtcp2_crypto_km **pckm, const uint8_t *secret, - size_t secretlen, const uint8_t *key, size_t keylen, + size_t secretlen, + const ngtcp2_crypto_aead_ctx *aead_ctx, const uint8_t *iv, size_t ivlen, const ngtcp2_mem *mem) { - int rv = ngtcp2_crypto_km_nocopy_new(pckm, secretlen, keylen, ivlen, mem); + int rv = ngtcp2_crypto_km_nocopy_new(pckm, secretlen, ivlen, mem); if (rv != 0) { return rv; } @@ -43,19 +44,20 @@ int ngtcp2_crypto_km_new(ngtcp2_crypto_km **pckm, const uint8_t *secret, if (secretlen) { memcpy((*pckm)->secret.base, secret, secretlen); } - memcpy((*pckm)->key.base, key, keylen); + if (aead_ctx) { + (*pckm)->aead_ctx = *aead_ctx; + } memcpy((*pckm)->iv.base, iv, ivlen); return 0; } int ngtcp2_crypto_km_nocopy_new(ngtcp2_crypto_km **pckm, size_t secretlen, - size_t keylen, size_t ivlen, - const ngtcp2_mem *mem) { + size_t ivlen, const ngtcp2_mem *mem) { size_t len; uint8_t *p; - len = sizeof(ngtcp2_crypto_km) + secretlen + keylen + ivlen; + len = sizeof(ngtcp2_crypto_km) + secretlen + ivlen; *pckm = ngtcp2_mem_malloc(mem, len); if (*pckm == NULL) { @@ -66,11 +68,9 @@ int ngtcp2_crypto_km_nocopy_new(ngtcp2_crypto_km **pckm, size_t secretlen, (*pckm)->secret.base = p; (*pckm)->secret.len = secretlen; p += secretlen; - (*pckm)->key.base = p; - (*pckm)->key.len = keylen; - p += keylen; (*pckm)->iv.base = p; (*pckm)->iv.len = ivlen; + (*pckm)->aead_ctx.native_handle = NULL; (*pckm)->pkt_num = -1; (*pckm)->use_count = 0; (*pckm)->flags = NGTCP2_CRYPTO_KM_FLAG_NONE; diff --git a/deps/ngtcp2/lib/ngtcp2_crypto.h b/deps/ngtcp2/lib/ngtcp2_crypto.h index 1cc144e801a1ef..b1baf30a85f6fc 100644 --- a/deps/ngtcp2/lib/ngtcp2_crypto.h +++ b/deps/ngtcp2/lib/ngtcp2_crypto.h @@ -50,7 +50,7 @@ typedef enum { typedef struct { ngtcp2_vec secret; - ngtcp2_vec key; + ngtcp2_crypto_aead_ctx aead_ctx; ngtcp2_vec iv; /* pkt_num is a packet number of a packet which uses this keying material. For encryption key, it is the lowest packet number of @@ -75,7 +75,8 @@ typedef struct { * store secret is update keys. Only 1RTT key can be updated. */ int ngtcp2_crypto_km_new(ngtcp2_crypto_km **pckm, const uint8_t *secret, - size_t secretlen, const uint8_t *key, size_t keylen, + size_t secretlen, + const ngtcp2_crypto_aead_ctx *aead_ctx, const uint8_t *iv, size_t ivlen, const ngtcp2_mem *mem); @@ -84,8 +85,7 @@ int ngtcp2_crypto_km_new(ngtcp2_crypto_km **pckm, const uint8_t *secret, * it does not copy secret, key and IV. */ int ngtcp2_crypto_km_nocopy_new(ngtcp2_crypto_km **pckm, size_t secretlen, - size_t keylen, size_t ivlen, - const ngtcp2_mem *mem); + size_t ivlen, const ngtcp2_mem *mem); void ngtcp2_crypto_km_del(ngtcp2_crypto_km *ckm, const ngtcp2_mem *mem); @@ -93,7 +93,7 @@ typedef struct { ngtcp2_crypto_aead aead; ngtcp2_crypto_cipher hp; ngtcp2_crypto_km *ckm; - const ngtcp2_vec *hp_key; + ngtcp2_crypto_cipher_ctx hp_ctx; size_t aead_overhead; ngtcp2_encrypt encrypt; ngtcp2_decrypt decrypt; diff --git a/deps/ngtcp2/lib/ngtcp2_err.c b/deps/ngtcp2/lib/ngtcp2_err.c index 6de827b0f97d2b..5a2425367d48a8 100644 --- a/deps/ngtcp2/lib/ngtcp2_err.c +++ b/deps/ngtcp2/lib/ngtcp2_err.c @@ -94,8 +94,8 @@ const char *ngtcp2_strerror(int liberr) { return "ERR_INTERNAL"; case NGTCP2_ERR_CRYPTO_BUFFER_EXCEEDED: return "ERR_CRYPTO_BUFFER_EXCEEDED"; - case NGTCP2_ERR_WRITE_STREAM_MORE: - return "ERR_WRITE_STREAM_MORE"; + case NGTCP2_ERR_WRITE_MORE: + return "ERR_WRITE_MORE"; case NGTCP2_ERR_RETRY: return "ERR_RETRY"; case NGTCP2_ERR_DROP_CONN: diff --git a/deps/ngtcp2/lib/ngtcp2_gaptr.c b/deps/ngtcp2/lib/ngtcp2_gaptr.c index b93cd828c211b3..6e7f3b7e554826 100644 --- a/deps/ngtcp2/lib/ngtcp2_gaptr.c +++ b/deps/ngtcp2/lib/ngtcp2_gaptr.c @@ -26,6 +26,7 @@ #include "ngtcp2_range.h" #include +#include int ngtcp2_gaptr_init(ngtcp2_gaptr *gaptr, const ngtcp2_mem *mem) { int rv; @@ -115,3 +116,14 @@ int ngtcp2_gaptr_is_pushed(ngtcp2_gaptr *gaptr, uint64_t offset, ngtcp2_range m = ngtcp2_range_intersect(&q, &k); return ngtcp2_range_len(&m) == 0; } + +void ngtcp2_gaptr_drop_first_gap(ngtcp2_gaptr *gaptr) { + ngtcp2_ksl_it it = ngtcp2_ksl_begin(&gaptr->gap); + ngtcp2_range r; + + assert(!ngtcp2_ksl_it_end(&it)); + + r = *(ngtcp2_range *)ngtcp2_ksl_it_key(&it); + + ngtcp2_ksl_remove(&gaptr->gap, NULL, &r); +} diff --git a/deps/ngtcp2/lib/ngtcp2_gaptr.h b/deps/ngtcp2/lib/ngtcp2_gaptr.h index 03181449deaf02..9e7fa03086e295 100644 --- a/deps/ngtcp2/lib/ngtcp2_gaptr.h +++ b/deps/ngtcp2/lib/ngtcp2_gaptr.h @@ -93,4 +93,11 @@ ngtcp2_ksl_it ngtcp2_gaptr_get_first_gap_after(ngtcp2_gaptr *gaptr, int ngtcp2_gaptr_is_pushed(ngtcp2_gaptr *gaptr, uint64_t offset, size_t datalen); +/* + * ngtcp2_gaptr_drop_first_gap deletes the first gap entirely as if + * the range is pushed. This function assumes that at least one gap + * exists. + */ +void ngtcp2_gaptr_drop_first_gap(ngtcp2_gaptr *gaptr); + #endif /* NGTCP2_GAPTR_H */ diff --git a/deps/ngtcp2/lib/ngtcp2_ksl.c b/deps/ngtcp2/lib/ngtcp2_ksl.c index 10c49806254890..0c47150cb143e7 100644 --- a/deps/ngtcp2/lib/ngtcp2_ksl.c +++ b/deps/ngtcp2/lib/ngtcp2_ksl.c @@ -275,6 +275,13 @@ int ngtcp2_ksl_insert(ngtcp2_ksl *ksl, ngtcp2_ksl_it *it, i = ksl_bsearch(ksl, blk, key, ksl->compar); if (blk->leaf) { + if (i < blk->n && + !ksl->compar(key, ngtcp2_ksl_nth_node(ksl, blk, i)->key)) { + if (it) { + *it = ngtcp2_ksl_end(ksl); + } + return NGTCP2_ERR_INVALID_ARGUMENT; + } ksl_insert_node(ksl, blk, i, key, data); ++ksl->n; if (it) { @@ -446,8 +453,8 @@ static int key_equal(ngtcp2_ksl_compar compar, const ngtcp2_ksl_key *lhs, return !compar(lhs, rhs) && !compar(rhs, lhs); } -void ngtcp2_ksl_remove(ngtcp2_ksl *ksl, ngtcp2_ksl_it *it, - const ngtcp2_ksl_key *key) { +int ngtcp2_ksl_remove(ngtcp2_ksl *ksl, ngtcp2_ksl_it *it, + const ngtcp2_ksl_key *key) { ngtcp2_ksl_blk *blk = ksl->head; ngtcp2_ksl_node *node; size_t i; @@ -461,10 +468,20 @@ void ngtcp2_ksl_remove(ngtcp2_ksl *ksl, ngtcp2_ksl_it *it, for (;;) { i = ksl_bsearch(ksl, blk, key, ksl->compar); - assert(i < blk->n); + if (i == blk->n) { + if (it) { + *it = ngtcp2_ksl_end(ksl); + } + return NGTCP2_ERR_INVALID_ARGUMENT; + } if (blk->leaf) { - assert(i < blk->n); + if (ksl->compar(key, ngtcp2_ksl_nth_node(ksl, blk, i)->key)) { + if (it) { + *it = ngtcp2_ksl_end(ksl); + } + return NGTCP2_ERR_INVALID_ARGUMENT; + } ksl_remove_node(ksl, blk, i); --ksl->n; if (it) { @@ -474,7 +491,7 @@ void ngtcp2_ksl_remove(ngtcp2_ksl *ksl, ngtcp2_ksl_it *it, ngtcp2_ksl_it_init(it, ksl, blk, i); } } - return; + return 0; } node = ngtcp2_ksl_nth_node(ksl, blk, i); diff --git a/deps/ngtcp2/lib/ngtcp2_ksl.h b/deps/ngtcp2/lib/ngtcp2_ksl.h index ae8bb8de5b5bf1..071e10dce1b40e 100644 --- a/deps/ngtcp2/lib/ngtcp2_ksl.h +++ b/deps/ngtcp2/lib/ngtcp2_ksl.h @@ -167,27 +167,33 @@ void ngtcp2_ksl_free(ngtcp2_ksl *ksl); * successful insertion, the iterator points to the inserted node is * stored in |*it|. * - * This function assumes that |key| does not exist in |ksl|. - * * This function returns 0 if it succeeds, or one of the following * negative error codes: * * NGTCP2_ERR_NOMEM * Out of memory. + * NGTCP2_ERR_INVALID_ARGUMENT + * |key| already exists. */ int ngtcp2_ksl_insert(ngtcp2_ksl *ksl, ngtcp2_ksl_it *it, const ngtcp2_ksl_key *key, void *data); /* - * ngtcp2_ksl_remove removes the |key| from |ksl|. It assumes such - * the key is included in |ksl|. + * ngtcp2_ksl_remove removes the |key| from |ksl|. * * This function assigns the iterator to |*it|, which points to the * node which is located at the right next of the removed node if |it| - * is not NULL. + * is not NULL. If |key| is not found, no deletion takes place and + * the return value of ngtcp2_ksl_end(ksl) is assigned to |*it|. + * + * This function returns 0 if it succeeds, or one of the following + * negative error codes: + * + * NGTCP2_ERR_INVALID_ARGUMENT + * |key| does not exist. */ -void ngtcp2_ksl_remove(ngtcp2_ksl *ksl, ngtcp2_ksl_it *it, - const ngtcp2_ksl_key *key); +int ngtcp2_ksl_remove(ngtcp2_ksl *ksl, ngtcp2_ksl_it *it, + const ngtcp2_ksl_key *key); /* * ngtcp2_ksl_lower_bound returns the iterator which points to the diff --git a/deps/ngtcp2/lib/ngtcp2_map.c b/deps/ngtcp2/lib/ngtcp2_map.c index bd214ff87b5046..12a20283b4c8e0 100644 --- a/deps/ngtcp2/lib/ngtcp2_map.c +++ b/deps/ngtcp2/lib/ngtcp2_map.c @@ -26,6 +26,7 @@ #include "ngtcp2_map.h" #include +#include #include "ngtcp2_conv.h" @@ -34,8 +35,7 @@ int ngtcp2_map_init(ngtcp2_map *map, const ngtcp2_mem *mem) { map->mem = mem; map->tablelen = INITIAL_TABLE_LENGTH; - map->table = - ngtcp2_mem_calloc(mem, map->tablelen, sizeof(ngtcp2_map_entry *)); + map->table = ngtcp2_mem_calloc(mem, map->tablelen, sizeof(ngtcp2_map_bucket)); if (map->table == NULL) { return NGTCP2_ERR_NOMEM; } @@ -45,20 +45,52 @@ int ngtcp2_map_init(ngtcp2_map *map, const ngtcp2_mem *mem) { return 0; } -void ngtcp2_map_free(ngtcp2_map *map) { ngtcp2_mem_free(map->mem, map->table); } +void ngtcp2_map_free(ngtcp2_map *map) { + size_t i; + ngtcp2_map_bucket *bkt; + + if (!map) { + return; + } + + for (i = 0; i < map->tablelen; ++i) { + bkt = &map->table[i]; + if (bkt->ksl) { + ngtcp2_ksl_free(bkt->ksl); + ngtcp2_mem_free(map->mem, bkt->ksl); + } + } + + ngtcp2_mem_free(map->mem, map->table); +} void ngtcp2_map_each_free(ngtcp2_map *map, int (*func)(ngtcp2_map_entry *entry, void *ptr), void *ptr) { uint32_t i; + ngtcp2_map_bucket *bkt; + ngtcp2_ksl_it it; + for (i = 0; i < map->tablelen; ++i) { - ngtcp2_map_entry *entry; - for (entry = map->table[i]; entry;) { - ngtcp2_map_entry *next = entry->next; - func(entry, ptr); - entry = next; + bkt = &map->table[i]; + + if (bkt->ptr) { + func(bkt->ptr, ptr); + bkt->ptr = NULL; + assert(bkt->ksl == NULL || ngtcp2_ksl_len(bkt->ksl) == 0); + continue; + } + + if (bkt->ksl) { + for (it = ngtcp2_ksl_begin(bkt->ksl); !ngtcp2_ksl_it_end(&it); + ngtcp2_ksl_it_next(&it)) { + func(ngtcp2_ksl_it_get(&it), ptr); + } + + ngtcp2_ksl_free(bkt->ksl); + ngtcp2_mem_free(map->mem, bkt->ksl); + bkt->ksl = NULL; } - map->table[i] = NULL; } } @@ -67,15 +99,29 @@ int ngtcp2_map_each(ngtcp2_map *map, void *ptr) { int rv; uint32_t i; + ngtcp2_map_bucket *bkt; + ngtcp2_ksl_it it; + for (i = 0; i < map->tablelen; ++i) { - ngtcp2_map_entry *entry, *next; - for (entry = map->table[i]; entry;) { - next = entry->next; - rv = func(entry, ptr); + bkt = &map->table[i]; + + if (bkt->ptr) { + rv = func(bkt->ptr, ptr); if (rv != 0) { return rv; } - entry = next; + assert(bkt->ksl == NULL || ngtcp2_ksl_len(bkt->ksl) == 0); + continue; + } + + if (bkt->ksl) { + for (it = ngtcp2_ksl_begin(bkt->ksl); !ngtcp2_ksl_it_end(&it); + ngtcp2_ksl_it_next(&it)) { + rv = func(ngtcp2_ksl_it_get(&it), ptr); + if (rv != 0) { + return rv; + } + } } } return 0; @@ -95,71 +141,124 @@ static uint32_t hash(key_type key, uint32_t mod) { p = (uint8_t *)&key; end = p + sizeof(key_type); - for (; p != end; ++p) { - h ^= *p; - h *= 0x01000193u; + for (; p != end;) { + h ^= *p++; + h += (h << 1) + (h << 4) + (h << 7) + (h << 8) + (h << 24); } return h & (mod - 1); } -static int insert(ngtcp2_map_entry **table, uint32_t tablelen, - ngtcp2_map_entry *entry) { +static int less(const ngtcp2_ksl_key *lhs, const ngtcp2_ksl_key *rhs) { + return *(key_type *)lhs < *(key_type *)rhs; +} + +static int map_insert(ngtcp2_map *map, ngtcp2_map_bucket *table, + uint32_t tablelen, ngtcp2_map_entry *entry) { uint32_t h = hash(entry->key, tablelen); - if (table[h] == NULL) { - table[h] = entry; - } else { - ngtcp2_map_entry *p; - /* We won't allow duplicated key, so check it out. */ - for (p = table[h]; p; p = p->next) { - if (p->key == entry->key) { - return NGTCP2_ERR_INVALID_ARGUMENT; - } + ngtcp2_map_bucket *bkt = &table[h]; + const ngtcp2_mem *mem = map->mem; + int rv; + + if (bkt->ptr == NULL && (bkt->ksl == NULL || ngtcp2_ksl_len(bkt->ksl) == 0)) { + bkt->ptr = entry; + return 0; + } + + if (!bkt->ksl) { + bkt->ksl = ngtcp2_mem_malloc(mem, sizeof(*bkt->ksl)); + if (bkt->ksl == NULL) { + return NGTCP2_ERR_NOMEM; } - entry->next = table[h]; - table[h] = entry; + ngtcp2_ksl_init(bkt->ksl, less, sizeof(key_type), mem); } - return 0; + + if (bkt->ptr) { + rv = ngtcp2_ksl_insert(bkt->ksl, NULL, &bkt->ptr->key, bkt->ptr); + if (rv != 0) { + return rv; + } + + bkt->ptr = NULL; + } + + return ngtcp2_ksl_insert(bkt->ksl, NULL, &entry->key, entry); } /* new_tablelen must be power of 2 */ -static int resize(ngtcp2_map *map, uint32_t new_tablelen) { +static int map_resize(ngtcp2_map *map, uint32_t new_tablelen) { uint32_t i; - ngtcp2_map_entry **new_table; + ngtcp2_map_bucket *new_table; + ngtcp2_map_bucket *bkt; + ngtcp2_ksl_it it; + int rv; new_table = - ngtcp2_mem_calloc(map->mem, new_tablelen, sizeof(ngtcp2_map_entry *)); + ngtcp2_mem_calloc(map->mem, new_tablelen, sizeof(ngtcp2_map_bucket)); if (new_table == NULL) { return NGTCP2_ERR_NOMEM; } for (i = 0; i < map->tablelen; ++i) { - ngtcp2_map_entry *entry; - for (entry = map->table[i]; entry;) { - ngtcp2_map_entry *next = entry->next; - entry->next = NULL; - /* This function must succeed */ - insert(new_table, new_tablelen, entry); - entry = next; + bkt = &map->table[i]; + + if (bkt->ptr) { + rv = map_insert(map, new_table, new_tablelen, bkt->ptr); + if (rv != 0) { + goto fail; + } + assert(bkt->ksl == NULL || ngtcp2_ksl_len(bkt->ksl) == 0); + continue; + } + + if (bkt->ksl) { + for (it = ngtcp2_ksl_begin(bkt->ksl); !ngtcp2_ksl_it_end(&it); + ngtcp2_ksl_it_next(&it)) { + rv = map_insert(map, new_table, new_tablelen, ngtcp2_ksl_it_get(&it)); + if (rv != 0) { + goto fail; + } + } + } + } + + for (i = 0; i < map->tablelen; ++i) { + bkt = &map->table[i]; + if (bkt->ksl) { + ngtcp2_ksl_free(bkt->ksl); + ngtcp2_mem_free(map->mem, bkt->ksl); } } + ngtcp2_mem_free(map->mem, map->table); map->tablelen = new_tablelen; map->table = new_table; return 0; + +fail: + for (i = 0; i < new_tablelen; ++i) { + bkt = &new_table[i]; + if (bkt->ksl) { + ngtcp2_ksl_free(bkt->ksl); + ngtcp2_mem_free(map->mem, bkt->ksl); + } + } + + return rv; } int ngtcp2_map_insert(ngtcp2_map *map, ngtcp2_map_entry *new_entry) { int rv; + /* Load factor is 0.75 */ if ((map->size + 1) * 4 > map->tablelen * 3) { - rv = resize(map, map->tablelen * 2); + rv = map_resize(map, map->tablelen * 2); if (rv != 0) { return rv; } } - rv = insert(map->table, map->tablelen, new_entry); + rv = map_insert(map, map->table, map->tablelen, new_entry); if (rv != 0) { return rv; } @@ -168,40 +267,64 @@ int ngtcp2_map_insert(ngtcp2_map *map, ngtcp2_map_entry *new_entry) { } ngtcp2_map_entry *ngtcp2_map_find(ngtcp2_map *map, key_type key) { - uint32_t h; - ngtcp2_map_entry *entry; - h = hash(key, map->tablelen); - for (entry = map->table[h]; entry; entry = entry->next) { - if (entry->key == key) { - return entry; + ngtcp2_map_bucket *bkt = &map->table[hash(key, map->tablelen)]; + ngtcp2_ksl_it it; + + if (bkt->ptr) { + if (bkt->ptr->key == key) { + return bkt->ptr; } + return NULL; } + + if (bkt->ksl) { + it = ngtcp2_ksl_lower_bound(bkt->ksl, &key); + if (ngtcp2_ksl_it_end(&it) || *(key_type *)ngtcp2_ksl_it_key(&it) != key) { + return NULL; + } + return ngtcp2_ksl_it_get(&it); + } + return NULL; } int ngtcp2_map_remove(ngtcp2_map *map, key_type key) { - uint32_t h; - ngtcp2_map_entry **dst; - - h = hash(key, map->tablelen); + ngtcp2_map_bucket *bkt = &map->table[hash(key, map->tablelen)]; + int rv; - for (dst = &map->table[h]; *dst; dst = &(*dst)->next) { - if ((*dst)->key != key) { - continue; + if (bkt->ptr) { + if (bkt->ptr->key == key) { + bkt->ptr = NULL; + --map->size; + return 0; } + return NGTCP2_ERR_INVALID_ARGUMENT; + } - *dst = (*dst)->next; + if (bkt->ksl) { + rv = ngtcp2_ksl_remove(bkt->ksl, NULL, &key); + if (rv != 0) { + return rv; + } --map->size; return 0; } + return NGTCP2_ERR_INVALID_ARGUMENT; } void ngtcp2_map_clear(ngtcp2_map *map) { uint32_t i; + ngtcp2_map_bucket *bkt; for (i = 0; i < map->tablelen; ++i) { - map->table[i] = NULL; + bkt = &map->table[i]; + bkt->ptr = NULL; + if (bkt->ksl) { + ngtcp2_ksl_free(bkt->ksl); + ngtcp2_mem_free(map->mem, bkt->ksl); + bkt->ksl = NULL; + } } map->size = 0; diff --git a/deps/ngtcp2/lib/ngtcp2_map.h b/deps/ngtcp2/lib/ngtcp2_map.h index 86f23a16a7e5dd..20afce24e9adab 100644 --- a/deps/ngtcp2/lib/ngtcp2_map.h +++ b/deps/ngtcp2/lib/ngtcp2_map.h @@ -33,6 +33,7 @@ #include #include "ngtcp2_mem.h" +#include "ngtcp2_ksl.h" /* Implementation of unordered map */ @@ -43,8 +44,13 @@ typedef struct ngtcp2_map_entry { key_type key; } ngtcp2_map_entry; +typedef struct ngtcp2_map_bucket { + ngtcp2_map_entry *ptr; + ngtcp2_ksl *ksl; +} ngtcp2_map_bucket; + typedef struct { - ngtcp2_map_entry **table; + ngtcp2_map_bucket *table; const ngtcp2_mem *mem; size_t size; uint32_t tablelen; diff --git a/deps/ngtcp2/lib/ngtcp2_path.c b/deps/ngtcp2/lib/ngtcp2_path.c index ebda947c6d6379..3f35f28ef6a394 100644 --- a/deps/ngtcp2/lib/ngtcp2_path.c +++ b/deps/ngtcp2/lib/ngtcp2_path.c @@ -44,12 +44,16 @@ int ngtcp2_path_eq(const ngtcp2_path *a, const ngtcp2_path *b) { ngtcp2_addr_eq(&a->remote, &b->remote); } -void ngtcp2_path_storage_init(ngtcp2_path_storage *ps, const void *local_addr, +void ngtcp2_path_storage_init(ngtcp2_path_storage *ps, + const struct sockaddr *local_addr, size_t local_addrlen, void *local_user_data, - const void *remote_addr, size_t remote_addrlen, - void *remote_user_data) { - ngtcp2_addr_init(&ps->path.local, ps->local_addrbuf, 0, local_user_data); - ngtcp2_addr_init(&ps->path.remote, ps->remote_addrbuf, 0, remote_user_data); + const struct sockaddr *remote_addr, + size_t remote_addrlen, void *remote_user_data) { + ngtcp2_addr_init(&ps->path.local, (const struct sockaddr *)&ps->local_addrbuf, + 0, local_user_data); + ngtcp2_addr_init(&ps->path.remote, + (const struct sockaddr *)&ps->remote_addrbuf, 0, + remote_user_data); ngtcp2_addr_copy_byte(&ps->path.local, local_addr, local_addrlen); ngtcp2_addr_copy_byte(&ps->path.remote, remote_addr, remote_addrlen); @@ -63,6 +67,8 @@ void ngtcp2_path_storage_init2(ngtcp2_path_storage *ps, } void ngtcp2_path_storage_zero(ngtcp2_path_storage *ps) { - ngtcp2_addr_init(&ps->path.local, ps->local_addrbuf, 0, NULL); - ngtcp2_addr_init(&ps->path.remote, ps->remote_addrbuf, 0, NULL); + ngtcp2_addr_init(&ps->path.local, (const struct sockaddr *)&ps->local_addrbuf, + 0, NULL); + ngtcp2_addr_init(&ps->path.remote, + (const struct sockaddr *)&ps->remote_addrbuf, 0, NULL); } diff --git a/deps/ngtcp2/lib/ngtcp2_pkt.c b/deps/ngtcp2/lib/ngtcp2_pkt.c index a67a376d364fd9..905af29a822602 100644 --- a/deps/ngtcp2/lib/ngtcp2_pkt.c +++ b/deps/ngtcp2/lib/ngtcp2_pkt.c @@ -474,6 +474,7 @@ ngtcp2_ssize ngtcp2_pkt_decode_frame(ngtcp2_frame *dest, const uint8_t *payload, return ngtcp2_pkt_decode_stop_sending_frame(&dest->stop_sending, payload, payloadlen); case NGTCP2_FRAME_ACK: + case NGTCP2_FRAME_ACK_ECN: return ngtcp2_pkt_decode_ack_frame(&dest->ack, payload, payloadlen); case NGTCP2_FRAME_PATH_CHALLENGE: return ngtcp2_pkt_decode_path_challenge_frame(&dest->path_challenge, @@ -2066,16 +2067,12 @@ ngtcp2_pkt_write_stateless_reset(uint8_t *dest, size_t destlen, return p - dest; } -static const uint8_t retry_key[] = - "\xcc\xce\x18\x7e\xd0\x9a\x09\xd0\x57\x28\x15\x5a\x6c\xb9\x6b\xe1"; -static const uint8_t retry_nonce[] = - "\xe5\x49\x30\xf9\x7f\x21\x36\xf0\x53\x0a\x8c\x1c"; - ngtcp2_ssize ngtcp2_pkt_write_retry(uint8_t *dest, size_t destlen, const ngtcp2_cid *dcid, const ngtcp2_cid *scid, const ngtcp2_cid *odcid, const uint8_t *token, size_t tokenlen, - ngtcp2_encrypt encrypt, const ngtcp2_crypto_aead *aead) { + ngtcp2_encrypt encrypt, const ngtcp2_crypto_aead *aead, + const ngtcp2_crypto_aead_ctx *aead_ctx) { ngtcp2_pkt_hd hd; uint8_t pseudo_retry[1500]; ngtcp2_ssize pseudo_retrylen; @@ -2106,8 +2103,10 @@ ngtcp2_pkt_write_retry(uint8_t *dest, size_t destlen, const ngtcp2_cid *dcid, } /* OpenSSL does not like NULL plaintext. */ - rv = encrypt(tag, aead, (const uint8_t *)"", 0, retry_key, retry_nonce, - sizeof(retry_nonce) - 1, pseudo_retry, (size_t)pseudo_retrylen); + rv = encrypt(tag, aead, aead_ctx, (const uint8_t *)"", 0, + (const uint8_t *)NGTCP2_RETRY_NONCE, + sizeof(NGTCP2_RETRY_NONCE) - 1, pseudo_retry, + (size_t)pseudo_retrylen); if (rv != 0) { return rv; } @@ -2160,7 +2159,8 @@ ngtcp2_ssize ngtcp2_pkt_encode_pseudo_retry( int ngtcp2_pkt_verify_retry_tag(const ngtcp2_pkt_retry *retry, const uint8_t *pkt, size_t pktlen, ngtcp2_encrypt encrypt, - const ngtcp2_crypto_aead *aead) { + const ngtcp2_crypto_aead *aead, + const ngtcp2_crypto_aead_ctx *aead_ctx) { uint8_t pseudo_retry[1500]; size_t pseudo_retrylen; uint8_t *p = pseudo_retry; @@ -2181,8 +2181,9 @@ int ngtcp2_pkt_verify_retry_tag(const ngtcp2_pkt_retry *retry, pseudo_retrylen = (size_t)(p - pseudo_retry); /* OpenSSL does not like NULL plaintext. */ - rv = encrypt(tag, aead, (const uint8_t *)"", 0, retry_key, retry_nonce, - sizeof(retry_nonce) - 1, pseudo_retry, pseudo_retrylen); + rv = encrypt(tag, aead, aead_ctx, (const uint8_t *)"", 0, + (const uint8_t *)NGTCP2_RETRY_NONCE, + sizeof(NGTCP2_RETRY_NONCE) - 1, pseudo_retry, pseudo_retrylen); if (rv != 0) { return rv; } @@ -2229,7 +2230,9 @@ size_t ngtcp2_pkt_stream_max_datalen(int64_t stream_id, uint64_t offset, size_t ngtcp2_pkt_crypto_max_datalen(uint64_t offset, size_t len, size_t left) { size_t n = 1 /* type */ + ngtcp2_put_varint_len(offset); - if (left <= n) { + /* CRYPTO frame must contain nonzero length data. Return -1 if + there is no space to write crypto data. */ + if (left <= n + 1) { return (size_t)-1; } diff --git a/deps/ngtcp2/lib/ngtcp2_pkt.h b/deps/ngtcp2/lib/ngtcp2_pkt.h index caf4152c045558..7348cb9a9d01d5 100644 --- a/deps/ngtcp2/lib/ngtcp2_pkt.h +++ b/deps/ngtcp2/lib/ngtcp2_pkt.h @@ -1120,6 +1120,7 @@ ngtcp2_ssize ngtcp2_pkt_encode_pseudo_retry( int ngtcp2_pkt_verify_retry_tag(const ngtcp2_pkt_retry *retry, const uint8_t *pkt, size_t pktlen, ngtcp2_encrypt encrypt, - const ngtcp2_crypto_aead *aead); + const ngtcp2_crypto_aead *aead, + const ngtcp2_crypto_aead_ctx *aead_ctx); #endif /* NGTCP2_PKT_H */ diff --git a/deps/ngtcp2/lib/ngtcp2_ppe.c b/deps/ngtcp2/lib/ngtcp2_ppe.c index c1856d0fe092e7..e2c47fd0c5fe05 100644 --- a/deps/ngtcp2/lib/ngtcp2_ppe.c +++ b/deps/ngtcp2/lib/ngtcp2_ppe.c @@ -123,7 +123,7 @@ ngtcp2_ssize ngtcp2_ppe_final(ngtcp2_ppe *ppe, const uint8_t **ppkt) { ngtcp2_crypto_create_nonce(ppe->nonce, cc->ckm->iv.base, cc->ckm->iv.len, ppe->pkt_num); - rv = cc->encrypt(payload, &cc->aead, payload, payloadlen, cc->ckm->key.base, + rv = cc->encrypt(payload, &cc->aead, &cc->ckm->aead_ctx, payload, payloadlen, ppe->nonce, cc->ckm->iv.len, buf->begin, ppe->hdlen); if (rv != 0) { return NGTCP2_ERR_CALLBACK_FAILURE; @@ -134,8 +134,7 @@ ngtcp2_ssize ngtcp2_ppe_final(ngtcp2_ppe *ppe, const uint8_t **ppkt) { /* TODO Check that we have enough space to get sample */ assert(ppe->sample_offset + NGTCP2_HP_SAMPLELEN <= ngtcp2_buf_len(buf)); - rv = cc->hp_mask(mask, &cc->hp, cc->hp_key->base, - buf->begin + ppe->sample_offset); + rv = cc->hp_mask(mask, &cc->hp, &cc->hp_ctx, buf->begin + ppe->sample_offset); if (rv != 0) { return NGTCP2_ERR_CALLBACK_FAILURE; } diff --git a/deps/ngtcp2/lib/ngtcp2_qlog.c b/deps/ngtcp2/lib/ngtcp2_qlog.c index f0abc8dba7ca1b..580b6a6fe0fd27 100644 --- a/deps/ngtcp2/lib/ngtcp2_qlog.c +++ b/deps/ngtcp2/lib/ngtcp2_qlog.c @@ -230,7 +230,8 @@ void ngtcp2_qlog_start(ngtcp2_qlog *qlog, const ngtcp2_cid *odcid, int server) { p = write_event_fields(p); p = write_events_start(p); - qlog->write(qlog->user_data, buf, (size_t)(p - buf)); + qlog->write(qlog->user_data, NGTCP2_QLOG_WRITE_FLAG_NONE, buf, + (size_t)(p - buf)); } void ngtcp2_qlog_end(ngtcp2_qlog *qlog) { @@ -245,7 +246,8 @@ void ngtcp2_qlog_end(ngtcp2_qlog *qlog) { p = write_trace_end(p); *p++ = '}'; - qlog->write(qlog->user_data, buf, (size_t)(p - buf)); + qlog->write(qlog->user_data, NGTCP2_QLOG_WRITE_FLAG_FIN, buf, + (size_t)(p - buf)); } static uint8_t *write_pkt_hd(uint8_t *p, const ngtcp2_pkt_hd *hd, @@ -814,7 +816,8 @@ static void qlog_pkt_write_end(ngtcp2_qlog *qlog, const ngtcp2_pkt_hd *hd, qlog->buf.last = p; - qlog->write(qlog->user_data, qlog->buf.pos, ngtcp2_buf_len(&qlog->buf)); + qlog->write(qlog->user_data, NGTCP2_QLOG_WRITE_FLAG_NONE, qlog->buf.pos, + ngtcp2_buf_len(&qlog->buf)); } void ngtcp2_qlog_write_frame(ngtcp2_qlog *qlog, const ngtcp2_frame *fr) { @@ -1015,7 +1018,7 @@ void ngtcp2_qlog_pkt_sent_end(ngtcp2_qlog *qlog, const ngtcp2_pkt_hd *hd, void ngtcp2_qlog_parameters_set_transport_params( ngtcp2_qlog *qlog, const ngtcp2_transport_params *params, int server, - int local) { + ngtcp2_qlog_side side) { uint8_t buf[1024]; uint8_t *p = buf; ngtcp2_vec name, value; @@ -1034,20 +1037,20 @@ void ngtcp2_qlog_parameters_set_transport_params( *p++ = ','; *p++ = '{'; p = write_pair(p, ngtcp2_vec_lit(&name, "owner"), - local ? ngtcp2_vec_lit(&value, "local") - : ngtcp2_vec_lit(&value, "remote")); + side == NGTCP2_QLOG_SIDE_LOCAL + ? ngtcp2_vec_lit(&value, "local") + : ngtcp2_vec_lit(&value, "remote")); *p++ = ','; p = write_pair_cid(p, ngtcp2_vec_lit(&name, "initial_source_connection_id"), ¶ms->initial_scid); *p++ = ','; - if (!server == !local) { + if (side == (server ? NGTCP2_QLOG_SIDE_LOCAL : NGTCP2_QLOG_SIDE_REMOTE)) { p = write_pair_cid( p, ngtcp2_vec_lit(&name, "original_destination_connection_id"), ¶ms->original_dcid); *p++ = ','; } if (params->retry_scid_present) { - assert(!server); p = write_pair_cid(p, ngtcp2_vec_lit(&name, "retry_source_connection_id"), ¶ms->retry_scid); *p++ = ','; @@ -1126,7 +1129,8 @@ void ngtcp2_qlog_parameters_set_transport_params( *p++ = ']'; *p++ = ','; - qlog->write(qlog->user_data, buf, (size_t)(p - buf)); + qlog->write(qlog->user_data, NGTCP2_QLOG_WRITE_FLAG_NONE, buf, + (size_t)(p - buf)); } void ngtcp2_qlog_metrics_updated(ngtcp2_qlog *qlog, @@ -1181,7 +1185,8 @@ void ngtcp2_qlog_metrics_updated(ngtcp2_qlog *qlog, *p++ = ']'; *p++ = ','; - qlog->write(qlog->user_data, buf, (size_t)(p - buf)); + qlog->write(qlog->user_data, NGTCP2_QLOG_WRITE_FLAG_NONE, buf, + (size_t)(p - buf)); } void ngtcp2_qlog_pkt_lost(ngtcp2_qlog *qlog, ngtcp2_rtb_entry *ent) { @@ -1215,5 +1220,6 @@ void ngtcp2_qlog_pkt_lost(ngtcp2_qlog *qlog, ngtcp2_rtb_entry *ent) { *p++ = ']'; *p++ = ','; - qlog->write(qlog->user_data, buf, (size_t)(p - buf)); + qlog->write(qlog->user_data, NGTCP2_QLOG_WRITE_FLAG_NONE, buf, + (size_t)(p - buf)); } diff --git a/deps/ngtcp2/lib/ngtcp2_qlog.h b/deps/ngtcp2/lib/ngtcp2_qlog.h index cd967984a7482a..0709f92a3648dc 100644 --- a/deps/ngtcp2/lib/ngtcp2_qlog.h +++ b/deps/ngtcp2/lib/ngtcp2_qlog.h @@ -40,6 +40,11 @@ qlog. */ #define NGTCP2_QLOG_BUFLEN 4096 +typedef enum ngtcp2_qlog_side { + NGTCP2_QLOG_SIDE_LOCAL, + NGTCP2_QLOG_SIDE_REMOTE, +} ngtcp2_qlog_side; + typedef struct ngtcp2_qlog { /* write is a callback function to write qlog. */ ngtcp2_qlog_write write; @@ -116,7 +121,7 @@ void ngtcp2_qlog_pkt_sent_end(ngtcp2_qlog *qlog, const ngtcp2_pkt_hd *hd, */ void ngtcp2_qlog_parameters_set_transport_params( ngtcp2_qlog *qlog, const ngtcp2_transport_params *params, int server, - int local); + ngtcp2_qlog_side side); /* * ngtcp2_qlog_metrics_updated writes metrics_updated event of diff --git a/deps/ngtcp2/lib/ngtcp2_rcvry.h b/deps/ngtcp2/lib/ngtcp2_rcvry.h index 0ff0bc4b6ed75e..e392c34ebfedb7 100644 --- a/deps/ngtcp2/lib/ngtcp2_rcvry.h +++ b/deps/ngtcp2/lib/ngtcp2_rcvry.h @@ -39,6 +39,4 @@ draft-ietf-quic-recovery-17. */ #define NGTCP2_GRANULARITY NGTCP2_MILLISECONDS -#define NGTCP2_DEFAULT_INITIAL_RTT (333 * NGTCP2_MILLISECONDS) - #endif /* NGTCP2_RCVRY_H */ diff --git a/deps/ngtcp2/lib/ngtcp2_ringbuf.c b/deps/ngtcp2/lib/ngtcp2_ringbuf.c index 6f5e61a2d69bd0..e4deab1ff76b83 100644 --- a/deps/ngtcp2/lib/ngtcp2_ringbuf.c +++ b/deps/ngtcp2/lib/ngtcp2_ringbuf.c @@ -31,6 +31,16 @@ #include "ngtcp2_macro.h" +#if defined(_MSC_VER) && defined(_M_ARM64) +unsigned int __popcnt(unsigned int x) { + unsigned int c = 0; + for (; x; ++c) { + x &= x - 1; + } + return c; +} +#endif + int ngtcp2_ringbuf_init(ngtcp2_ringbuf *rb, size_t nmemb, size_t size, const ngtcp2_mem *mem) { #ifdef WIN32 @@ -101,6 +111,4 @@ void *ngtcp2_ringbuf_get(ngtcp2_ringbuf *rb, size_t offset) { return &rb->buf[offset * rb->size]; } -size_t ngtcp2_ringbuf_len(ngtcp2_ringbuf *rb) { return rb->len; } - int ngtcp2_ringbuf_full(ngtcp2_ringbuf *rb) { return rb->len == rb->nmemb; } diff --git a/deps/ngtcp2/lib/ngtcp2_ringbuf.h b/deps/ngtcp2/lib/ngtcp2_ringbuf.h index 05e9a577f1948e..6d546495f2ac51 100644 --- a/deps/ngtcp2/lib/ngtcp2_ringbuf.h +++ b/deps/ngtcp2/lib/ngtcp2_ringbuf.h @@ -102,7 +102,7 @@ void ngtcp2_ringbuf_resize(ngtcp2_ringbuf *rb, size_t len); void *ngtcp2_ringbuf_get(ngtcp2_ringbuf *rb, size_t offset); /* ngtcp2_ringbuf_len returns the number of elements stored. */ -size_t ngtcp2_ringbuf_len(ngtcp2_ringbuf *rb); +#define ngtcp2_ringbuf_len(RB) ((RB)->len) /* ngtcp2_ringbuf_full returns nonzero if |rb| is full. */ int ngtcp2_ringbuf_full(ngtcp2_ringbuf *rb); diff --git a/deps/ngtcp2/lib/ngtcp2_rtb.c b/deps/ngtcp2/lib/ngtcp2_rtb.c index e7e6762e03daf4..d58a6b2ea1adb7 100644 --- a/deps/ngtcp2/lib/ngtcp2_rtb.c +++ b/deps/ngtcp2/lib/ngtcp2_rtb.c @@ -84,11 +84,53 @@ int ngtcp2_frame_chain_crypto_datacnt_new(ngtcp2_frame_chain **pfrc, return ngtcp2_frame_chain_new(pfrc, mem); } +int ngtcp2_frame_chain_new_token_new(ngtcp2_frame_chain **pfrc, + const ngtcp2_vec *token, + const ngtcp2_mem *mem) { + size_t avail = sizeof(ngtcp2_frame) - sizeof(ngtcp2_new_token); + int rv; + uint8_t *p; + ngtcp2_frame *fr; + + if (token->len > avail) { + rv = ngtcp2_frame_chain_extralen_new(pfrc, token->len - avail, mem); + } else { + rv = ngtcp2_frame_chain_new(pfrc, mem); + } + if (rv != 0) { + return rv; + } + + fr = &(*pfrc)->fr; + fr->type = NGTCP2_FRAME_NEW_TOKEN; + + p = (uint8_t *)(*pfrc) + sizeof(ngtcp2_new_token); + memcpy(p, token->base, token->len); + + ngtcp2_vec_init(&fr->new_token.token, p, token->len); + + return 0; +} + void ngtcp2_frame_chain_del(ngtcp2_frame_chain *frc, const ngtcp2_mem *mem) { + ngtcp2_frame_chain_binder *binder; + + if (frc == NULL) { + return; + } + + binder = frc->binder; + if (binder && --binder->refcount == 0) { + ngtcp2_mem_free(mem, binder); + } + ngtcp2_mem_free(mem, frc); } -void ngtcp2_frame_chain_init(ngtcp2_frame_chain *frc) { frc->next = NULL; } +void ngtcp2_frame_chain_init(ngtcp2_frame_chain *frc) { + frc->next = NULL; + frc->binder = NULL; +} void ngtcp2_frame_chain_list_del(ngtcp2_frame_chain *frc, const ngtcp2_mem *mem) { @@ -96,22 +138,42 @@ void ngtcp2_frame_chain_list_del(ngtcp2_frame_chain *frc, for (; frc;) { next = frc->next; - ngtcp2_mem_free(mem, frc); + ngtcp2_frame_chain_del(frc, mem); frc = next; } } -static void frame_chain_insert(ngtcp2_frame_chain **pfrc, - ngtcp2_frame_chain *frc) { - ngtcp2_frame_chain **plast; +int ngtcp2_frame_chain_binder_new(ngtcp2_frame_chain_binder **pbinder, + const ngtcp2_mem *mem) { + *pbinder = ngtcp2_mem_calloc(mem, 1, sizeof(ngtcp2_frame_chain_binder)); + if (*pbinder == NULL) { + return NGTCP2_ERR_NOMEM; + } + + return 0; +} - assert(frc); +int ngtcp2_bind_frame_chains(ngtcp2_frame_chain *a, ngtcp2_frame_chain *b, + const ngtcp2_mem *mem) { + ngtcp2_frame_chain_binder *binder; + int rv; + + assert(b->binder == NULL); + + if (a->binder == NULL) { + rv = ngtcp2_frame_chain_binder_new(&binder, mem); + if (rv != 0) { + return rv; + } + + a->binder = binder; + ++a->binder->refcount; + } - for (plast = &frc; *plast; plast = &(*plast)->next) - ; + b->binder = a->binder; + ++b->binder->refcount; - *plast = *pfrc; - *pfrc = frc; + return 0; } int ngtcp2_rtb_entry_new(ngtcp2_rtb_entry **pent, const ngtcp2_pkt_hd *hd, @@ -127,8 +189,10 @@ int ngtcp2_rtb_entry_new(ngtcp2_rtb_entry **pent, const ngtcp2_pkt_hd *hd, (*pent)->hd.flags = hd->flags; (*pent)->frc = frc; (*pent)->ts = ts; + (*pent)->lost_ts = UINT64_MAX; (*pent)->pktlen = pktlen; (*pent)->flags = flags; + (*pent)->next = NULL; return 0; } @@ -160,10 +224,13 @@ void ngtcp2_rtb_init(ngtcp2_rtb *rtb, ngtcp2_pktns_id pktns_id, rtb->mem = mem; rtb->largest_acked_tx_pkt_num = -1; rtb->num_ack_eliciting = 0; + rtb->num_retransmittable = 0; rtb->probe_pkt_left = 0; rtb->pktns_id = pktns_id; rtb->cc_pkt_num = 0; rtb->cc_bytes_in_flight = 0; + rtb->persistent_congestion_start_ts = UINT64_MAX; + rtb->num_lost_pkts = 0; } void ngtcp2_rtb_free(ngtcp2_rtb *rtb) { @@ -196,15 +263,28 @@ static void rtb_on_add(ngtcp2_rtb *rtb, ngtcp2_rtb_entry *ent, if (ent->flags & NGTCP2_RTB_FLAG_ACK_ELICITING) { ++rtb->num_ack_eliciting; } + if (ent->flags & NGTCP2_RTB_FLAG_RETRANSMITTABLE) { + ++rtb->num_retransmittable; + } } static void rtb_on_remove(ngtcp2_rtb *rtb, ngtcp2_rtb_entry *ent, ngtcp2_conn_stat *cstat) { + if (ent->flags & NGTCP2_RTB_FLAG_LOST_RETRANSMITTED) { + return; + } + if (ent->flags & NGTCP2_RTB_FLAG_ACK_ELICITING) { assert(rtb->num_ack_eliciting); --rtb->num_ack_eliciting; } + if ((ent->flags & NGTCP2_RTB_FLAG_RETRANSMITTABLE) && + !(ent->flags & NGTCP2_RTB_FLAG_PTO_RECLAIMED)) { + assert(rtb->num_retransmittable); + --rtb->num_retransmittable; + } + if (rtb->cc_pkt_num <= ent->hd.pkt_num) { assert(cstat->bytes_in_flight >= ent->pktlen); cstat->bytes_in_flight -= ent->pktlen; @@ -214,8 +294,154 @@ static void rtb_on_remove(ngtcp2_rtb *rtb, ngtcp2_rtb_entry *ent, } } -static void rtb_on_pkt_lost(ngtcp2_rtb *rtb, ngtcp2_frame_chain **pfrc, - ngtcp2_rtb_entry *ent) { +/* + * rtb_reclaim_frame queues unacknowledged frames included in |ent| + * for retransmission. The re-queued frames are not deleted from + * |ent|. It returns the number of frames queued. + */ +static ngtcp2_ssize rtb_reclaim_frame(ngtcp2_rtb *rtb, ngtcp2_conn *conn, + ngtcp2_pktns *pktns, + ngtcp2_rtb_entry *ent) { + ngtcp2_frame_chain *frc, *nfrc, **pfrc = &pktns->tx.frq; + ngtcp2_frame *fr; + ngtcp2_strm *strm; + ngtcp2_range gap, range; + size_t num_reclaimed = 0; + int rv; + + /* PADDING only (or PADDING + ACK ) packets will have NULL + ent->frc. */ + /* TODO Reconsider the order of pfrc */ + for (frc = ent->frc; frc; frc = frc->next) { + fr = &frc->fr; + /* Check that a late ACK acknowledged this frame. */ + if (frc->binder && + (frc->binder->flags & NGTCP2_FRAME_CHAIN_BINDER_FLAG_ACK)) { + continue; + } + switch (frc->fr.type) { + case NGTCP2_FRAME_STREAM: + strm = ngtcp2_conn_find_stream(conn, fr->stream.stream_id); + if (strm == NULL) { + continue; + } + + gap = ngtcp2_strm_get_unacked_range_after(strm, fr->stream.offset); + + range.begin = fr->stream.offset; + range.end = fr->stream.offset + + ngtcp2_vec_len(fr->stream.data, fr->stream.datacnt); + range = ngtcp2_range_intersect(&range, &gap); + if (ngtcp2_range_len(&range) == 0 && + (!fr->stream.fin || (strm->flags & NGTCP2_STRM_FLAG_FIN_ACKED))) { + continue; + } + + rv = ngtcp2_frame_chain_stream_datacnt_new(&nfrc, fr->stream.datacnt, + rtb->mem); + if (rv != 0) { + return rv; + } + + nfrc->fr = *fr; + ngtcp2_vec_copy(nfrc->fr.stream.data, fr->stream.data, + fr->stream.datacnt); + + rv = ngtcp2_strm_streamfrq_push(strm, nfrc); + if (rv != 0) { + ngtcp2_frame_chain_del(nfrc, conn->mem); + return rv; + } + if (!ngtcp2_strm_is_tx_queued(strm)) { + strm->cycle = ngtcp2_conn_tx_strmq_first_cycle(conn); + rv = ngtcp2_conn_tx_strmq_push(conn, strm); + if (rv != 0) { + return rv; + } + } + + ++num_reclaimed; + + continue; + case NGTCP2_FRAME_CRYPTO: + /* Don't resend CRYPTO frame if the whole region it contains has + been acknowledged */ + gap = ngtcp2_strm_get_unacked_range_after(rtb->crypto, fr->crypto.offset); + + range.begin = fr->crypto.offset; + range.end = fr->crypto.offset + + ngtcp2_vec_len(fr->crypto.data, fr->crypto.datacnt); + range = ngtcp2_range_intersect(&range, &gap); + if (ngtcp2_range_len(&range) == 0) { + continue; + } + + rv = ngtcp2_frame_chain_crypto_datacnt_new(&nfrc, fr->crypto.datacnt, + rtb->mem); + if (rv != 0) { + return rv; + } + + nfrc->fr = *fr; + ngtcp2_vec_copy(nfrc->fr.crypto.data, fr->crypto.data, + fr->crypto.datacnt); + + rv = ngtcp2_ksl_insert(&pktns->crypto.tx.frq, NULL, + &nfrc->fr.crypto.offset, nfrc); + if (rv != 0) { + assert(ngtcp2_err_is_fatal(rv)); + ngtcp2_frame_chain_del(nfrc, conn->mem); + return rv; + } + + ++num_reclaimed; + + continue; + case NGTCP2_FRAME_NEW_TOKEN: + rv = ngtcp2_frame_chain_new_token_new(&nfrc, &fr->new_token.token, + rtb->mem); + if (rv != 0) { + return rv; + } + + rv = ngtcp2_bind_frame_chains(frc, nfrc, rtb->mem); + if (rv != 0) { + return rv; + } + + break; + default: + rv = ngtcp2_frame_chain_new(&nfrc, rtb->mem); + if (rv != 0) { + return rv; + } + + nfrc->fr = *fr; + + rv = ngtcp2_bind_frame_chains(frc, nfrc, rtb->mem); + if (rv != 0) { + return rv; + } + + break; + } + + ++num_reclaimed; + + nfrc->next = *pfrc; + *pfrc = nfrc; + pfrc = &nfrc->next; + } + + return (ngtcp2_ssize)num_reclaimed; +} + +static int rtb_on_pkt_lost(ngtcp2_rtb *rtb, ngtcp2_ksl_it *it, + ngtcp2_rtb_entry *ent, ngtcp2_conn *conn, + ngtcp2_pktns *pktns, ngtcp2_tstamp ts) { + int rv; + ngtcp2_ssize reclaimed; + ngtcp2_log_pkt_lost(rtb->log, ent->hd.pkt_num, ent->hd.type, ent->hd.flags, ent->ts); @@ -224,16 +450,42 @@ static void rtb_on_pkt_lost(ngtcp2_rtb *rtb, ngtcp2_frame_chain **pfrc, } if (!(ent->flags & NGTCP2_RTB_FLAG_PROBE)) { - if (ent->flags & NGTCP2_RTB_FLAG_CRYPTO_TIMEOUT_RETRANSMITTED) { + if (ent->flags & NGTCP2_RTB_FLAG_PTO_RECLAIMED) { ngtcp2_log_info(rtb->log, NGTCP2_LOG_EVENT_RCV, - "pkn=%" PRId64 " CRYPTO has already been retransmitted", + "pkn=%" PRId64 " has already been reclaimed on PTO", ent->hd.pkt_num); - } else if (ent->frc) { - /* PADDING only (or PADDING + ACK ) packets will have NULL - ent->frc. */ - /* TODO Reconsider the order of pfrc */ - frame_chain_insert(pfrc, ent->frc); - ent->frc = NULL; + assert(!(ent->flags & NGTCP2_RTB_FLAG_LOST_RETRANSMITTED)); + assert(UINT64_MAX == ent->lost_ts); + + ent->flags |= NGTCP2_RTB_FLAG_LOST_RETRANSMITTED; + ent->lost_ts = ts; + + ++rtb->num_lost_pkts; + + ngtcp2_ksl_it_next(it); + + return 0; + } + + if (ent->frc) { + assert(!(ent->flags & NGTCP2_RTB_FLAG_LOST_RETRANSMITTED)); + assert(UINT64_MAX == ent->lost_ts); + + reclaimed = rtb_reclaim_frame(rtb, conn, pktns, ent); + if (reclaimed < 0) { + return (int)reclaimed; + } + + if (reclaimed) { + ent->flags |= NGTCP2_RTB_FLAG_LOST_RETRANSMITTED; + ent->lost_ts = ts; + + ++rtb->num_lost_pkts; + + ngtcp2_ksl_it_next(it); + + return 0; + } } } else { ngtcp2_log_info(rtb->log, NGTCP2_LOG_EVENT_RCV, @@ -242,15 +494,18 @@ static void rtb_on_pkt_lost(ngtcp2_rtb *rtb, ngtcp2_frame_chain **pfrc, ent->hd.pkt_num); } + rv = ngtcp2_ksl_remove(&rtb->ents, it, &ent->hd.pkt_num); + assert(0 == rv); + ngtcp2_rtb_entry_del(ent, rtb->mem); + + return 0; } int ngtcp2_rtb_add(ngtcp2_rtb *rtb, ngtcp2_rtb_entry *ent, ngtcp2_conn_stat *cstat) { int rv; - ent->next = NULL; - rv = ngtcp2_ksl_insert(&rtb->ents, NULL, &ent->hd.pkt_num, ent); if (rv != 0) { return rv; @@ -266,10 +521,16 @@ ngtcp2_ksl_it ngtcp2_rtb_head(ngtcp2_rtb *rtb) { } static void rtb_remove(ngtcp2_rtb *rtb, ngtcp2_ksl_it *it, - ngtcp2_rtb_entry *ent, ngtcp2_conn_stat *cstat) { - ngtcp2_ksl_remove(&rtb->ents, it, &ent->hd.pkt_num); + ngtcp2_rtb_entry **pent, ngtcp2_rtb_entry *ent, + ngtcp2_conn_stat *cstat) { + int rv; + rv = ngtcp2_ksl_remove(&rtb->ents, it, &ent->hd.pkt_num); + assert(0 == rv); rtb_on_remove(rtb, ent, cstat); - ngtcp2_rtb_entry_del(ent, rtb->mem); + + assert(ent->next == NULL); + + ngtcp2_list_insert(ent, pent); } static int rtb_process_acked_pkt(ngtcp2_rtb *rtb, ngtcp2_rtb_entry *ent, @@ -283,23 +544,31 @@ static int rtb_process_acked_pkt(ngtcp2_rtb *rtb, ngtcp2_rtb_entry *ent, ngtcp2_crypto_level crypto_level; for (frc = ent->frc; frc; frc = frc->next) { + if (frc->binder) { + frc->binder->flags |= NGTCP2_FRAME_CHAIN_BINDER_FLAG_ACK; + } + switch (frc->fr.type) { case NGTCP2_FRAME_STREAM: strm = ngtcp2_conn_find_stream(conn, frc->fr.stream.stream_id); if (strm == NULL) { break; } - prev_stream_offset = - ngtcp2_gaptr_first_gap_offset(&strm->tx.acked_offset); - rv = ngtcp2_gaptr_push( - &strm->tx.acked_offset, frc->fr.stream.offset, + + if (frc->fr.stream.fin) { + strm->flags |= NGTCP2_STRM_FLAG_FIN_ACKED; + } + + prev_stream_offset = ngtcp2_strm_get_acked_offset(strm); + rv = ngtcp2_strm_ack_data( + strm, frc->fr.stream.offset, ngtcp2_vec_len(frc->fr.stream.data, frc->fr.stream.datacnt)); if (rv != 0) { return rv; } if (conn->callbacks.acked_stream_data_offset) { - stream_offset = ngtcp2_gaptr_first_gap_offset(&strm->tx.acked_offset); + stream_offset = ngtcp2_strm_get_acked_offset(strm); datalen = stream_offset - prev_stream_offset; if (datalen == 0 && !frc->fr.stream.fin) { break; @@ -319,17 +588,16 @@ static int rtb_process_acked_pkt(ngtcp2_rtb *rtb, ngtcp2_rtb_entry *ent, } break; case NGTCP2_FRAME_CRYPTO: - prev_stream_offset = - ngtcp2_gaptr_first_gap_offset(&crypto->tx.acked_offset); - rv = ngtcp2_gaptr_push( - &crypto->tx.acked_offset, frc->fr.crypto.offset, + prev_stream_offset = ngtcp2_strm_get_acked_offset(crypto); + rv = ngtcp2_strm_ack_data( + crypto, frc->fr.crypto.offset, ngtcp2_vec_len(frc->fr.crypto.data, frc->fr.crypto.datacnt)); if (rv != 0) { return rv; } if (conn->callbacks.acked_crypto_offset) { - stream_offset = ngtcp2_gaptr_first_gap_offset(&crypto->tx.acked_offset); + stream_offset = ngtcp2_strm_get_acked_offset(crypto); datalen = stream_offset - prev_stream_offset; if (datalen == 0) { break; @@ -403,11 +671,11 @@ ngtcp2_ssize ngtcp2_rtb_recv_ack(ngtcp2_rtb *rtb, const ngtcp2_ack *fr, int rv; ngtcp2_ksl_it it; ngtcp2_ssize num_acked = 0; - int largest_pkt_acked = 0; - int rtt_updated = 0; - ngtcp2_tstamp largest_pkt_sent_ts = 0; + ngtcp2_tstamp largest_pkt_sent_ts = UINT64_MAX; int64_t pkt_num; ngtcp2_cc *cc = rtb->cc; + ngtcp2_rtb_entry *acked_ent = NULL; + int ack_eliciting_pkt_acked = 0; if (conn && (conn->flags & NGTCP2_CONN_FLAG_KEY_UPDATE_NOT_CONFIRMED) && largest_ack >= conn->pktns.crypto.tx.ckm->pkt_num) { @@ -430,36 +698,25 @@ ngtcp2_ssize ngtcp2_rtb_recv_ack(ngtcp2_rtb *rtb, const ngtcp2_ack *fr, for (; !ngtcp2_ksl_it_end(&it);) { pkt_num = *(int64_t *)ngtcp2_ksl_it_key(&it); - if (min_ack <= pkt_num && pkt_num <= largest_ack) { - ent = ngtcp2_ksl_it_get(&it); - if (conn) { - rv = rtb_process_acked_pkt(rtb, ent, conn); - if (rv != 0) { - return rv; - } - if (largest_ack == pkt_num) { - largest_pkt_sent_ts = ent->ts; - largest_pkt_acked = 1; - } - if (!rtt_updated && largest_pkt_acked && - (ent->flags & NGTCP2_RTB_FLAG_ACK_ELICITING)) { - rtt_updated = 1; - ngtcp2_conn_update_rtt(conn, pkt_ts - largest_pkt_sent_ts, - fr->ack_delay_unscaled); - if (cc->new_rtt_sample) { - cc->new_rtt_sample(cc, cstat, ts); - } - } - rtb_on_pkt_acked(rtb, ent, cstat, ts); - /* At this point, it is invalided because rtb->ents might be - modified. */ - } - rtb_remove(rtb, &it, ent, cstat); - ++num_acked; - continue; + assert(pkt_num <= largest_ack); + + if (pkt_num < min_ack) { + break; + } + + ent = ngtcp2_ksl_it_get(&it); + + if (largest_ack == pkt_num) { + largest_pkt_sent_ts = ent->ts; } - break; + + if (ent->flags & NGTCP2_RTB_FLAG_ACK_ELICITING) { + ack_eliciting_pkt_acked = 1; + } + + rtb_remove(rtb, &it, &acked_ent, ent, cstat); + ++num_acked; } for (i = 0; i < fr->num_blks;) { @@ -477,44 +734,65 @@ ngtcp2_ssize ngtcp2_rtb_recv_ack(ngtcp2_rtb *rtb, const ngtcp2_ack *fr, break; } ent = ngtcp2_ksl_it_get(&it); - if (conn) { - rv = rtb_process_acked_pkt(rtb, ent, conn); - if (rv != 0) { - return rv; - } - if (!rtt_updated && largest_pkt_acked && - (ent->flags & NGTCP2_RTB_FLAG_ACK_ELICITING)) { - rtt_updated = 1; - ngtcp2_conn_update_rtt(conn, pkt_ts - largest_pkt_sent_ts, - fr->ack_delay_unscaled); - if (cc->new_rtt_sample) { - cc->new_rtt_sample(cc, cstat, ts); - } - } - rtb_on_pkt_acked(rtb, ent, cstat, ts); + if (ent->flags & NGTCP2_RTB_FLAG_ACK_ELICITING) { + ack_eliciting_pkt_acked = 1; } - rtb_remove(rtb, &it, ent, cstat); + + rtb_remove(rtb, &it, &acked_ent, ent, cstat); ++num_acked; } ++i; } + if (largest_pkt_sent_ts != UINT64_MAX && ack_eliciting_pkt_acked) { + ngtcp2_conn_update_rtt(conn, pkt_ts - largest_pkt_sent_ts, + fr->ack_delay_unscaled); + if (cc->new_rtt_sample) { + cc->new_rtt_sample(cc, cstat, ts); + } + } + ngtcp2_rst_on_ack_recv(rtb->rst, cstat); + + if (conn) { + for (ent = acked_ent; ent; ent = acked_ent) { + rv = rtb_process_acked_pkt(rtb, ent, conn); + if (rv != 0) { + goto fail; + } + + rtb_on_pkt_acked(rtb, ent, cstat, ts); + acked_ent = ent->next; + ngtcp2_rtb_entry_del(ent, rtb->mem); + } + } else { + /* For unit tests */ + for (ent = acked_ent; ent; ent = acked_ent) { + rtb_on_pkt_acked(rtb, ent, cstat, ts); + acked_ent = ent->next; + ngtcp2_rtb_entry_del(ent, rtb->mem); + } + } + cc->on_ack_recv(cc, cstat, ts); return num_acked; + +fail: + for (ent = acked_ent; ent; ent = acked_ent) { + acked_ent = ent->next; + ngtcp2_rtb_entry_del(ent, rtb->mem); + } + + return rv; } static int rtb_pkt_lost(ngtcp2_rtb *rtb, ngtcp2_conn_stat *cstat, const ngtcp2_rtb_entry *ent, uint64_t loss_delay, - ngtcp2_tstamp lost_send_time) { + ngtcp2_tstamp lost_send_time, uint64_t pkt_thres) { ngtcp2_tstamp loss_time; - uint64_t pkt_thres = - rtb->cc_bytes_in_flight / cstat->max_udp_payload_size / 2; - - pkt_thres = ngtcp2_max(pkt_thres, NGTCP2_PKT_THRESHOLD); if (ent->ts <= lost_send_time || rtb->largest_acked_tx_pkt_num >= ent->hd.pkt_num + (int64_t)pkt_thres) { @@ -535,8 +813,7 @@ static int rtb_pkt_lost(ngtcp2_rtb *rtb, ngtcp2_conn_stat *cstat, } /* - * rtb_compute_pkt_loss_delay computes delay until packet is - * considered lost in NGTCP2_MICROSECONDS resolution. + * rtb_compute_pkt_loss_delay computes loss delay. */ static ngtcp2_duration compute_pkt_loss_delay(const ngtcp2_conn_stat *cstat) { /* 9/8 is kTimeThreshold */ @@ -545,9 +822,9 @@ static ngtcp2_duration compute_pkt_loss_delay(const ngtcp2_conn_stat *cstat) { return ngtcp2_max(loss_delay, NGTCP2_GRANULARITY); } -void ngtcp2_rtb_detect_lost_pkt(ngtcp2_rtb *rtb, ngtcp2_frame_chain **pfrc, - ngtcp2_conn_stat *cstat, ngtcp2_duration pto, - ngtcp2_tstamp ts) { +int ngtcp2_rtb_detect_lost_pkt(ngtcp2_rtb *rtb, ngtcp2_conn *conn, + ngtcp2_pktns *pktns, ngtcp2_conn_stat *cstat, + ngtcp2_duration pto, ngtcp2_tstamp ts) { ngtcp2_rtb_entry *ent; ngtcp2_duration loss_delay; ngtcp2_tstamp lost_send_time; @@ -556,7 +833,11 @@ void ngtcp2_rtb_detect_lost_pkt(ngtcp2_rtb *rtb, ngtcp2_frame_chain **pfrc, int64_t last_lost_pkt_num; ngtcp2_duration loss_window, congestion_period; ngtcp2_cc *cc = rtb->cc; + int rv; + uint64_t pkt_thres = + rtb->cc_bytes_in_flight / cstat->max_udp_payload_size / 2; + pkt_thres = ngtcp2_max(pkt_thres, NGTCP2_PKT_THRESHOLD); cstat->loss_time[rtb->pktns_id] = UINT64_MAX; loss_delay = compute_pkt_loss_delay(cstat); lost_send_time = ts - loss_delay; @@ -565,31 +846,55 @@ void ngtcp2_rtb_detect_lost_pkt(ngtcp2_rtb *rtb, ngtcp2_frame_chain **pfrc, for (; !ngtcp2_ksl_it_end(&it); ngtcp2_ksl_it_next(&it)) { ent = ngtcp2_ksl_it_get(&it); - if (rtb_pkt_lost(rtb, cstat, ent, loss_delay, lost_send_time)) { + if (ent->flags & NGTCP2_RTB_FLAG_LOST_RETRANSMITTED) { + break; + } + + if (rtb_pkt_lost(rtb, cstat, ent, loss_delay, lost_send_time, pkt_thres)) { /* All entries from ent are considered to be lost. */ latest_ts = oldest_ts = ent->ts; last_lost_pkt_num = ent->hd.pkt_num; + congestion_period = pto * NGTCP2_PERSISTENT_CONGESTION_THRESHOLD; + for (; !ngtcp2_ksl_it_end(&it);) { ent = ngtcp2_ksl_it_get(&it); - ngtcp2_ksl_remove(&rtb->ents, &it, &ent->hd.pkt_num); - if (last_lost_pkt_num == ent->hd.pkt_num + 1) { + if (last_lost_pkt_num == ent->hd.pkt_num + 1 && + ent->ts >= rtb->persistent_congestion_start_ts) { last_lost_pkt_num = ent->hd.pkt_num; + oldest_ts = ent->ts; } else { last_lost_pkt_num = -1; } - oldest_ts = ent->ts; + if ((ent->flags & NGTCP2_RTB_FLAG_LOST_RETRANSMITTED)) { + if (rtb->pktns_id != NGTCP2_PKTNS_ID_APP || last_lost_pkt_num == -1 || + latest_ts - oldest_ts >= congestion_period) { + break; + } + ngtcp2_ksl_it_next(&it); + continue; + } + rtb_on_remove(rtb, ent, cstat); - rtb_on_pkt_lost(rtb, pfrc, ent); + rv = rtb_on_pkt_lost(rtb, &it, ent, conn, pktns, ts); + if (rv != 0) { + return rv; + } } cc->congestion_event(cc, cstat, latest_ts, ts); - if (last_lost_pkt_num != -1) { - loss_window = latest_ts - oldest_ts; - congestion_period = pto * NGTCP2_PERSISTENT_CONGESTION_THRESHOLD; + loss_window = latest_ts - oldest_ts; + /* Persistent congestion situation is only evaluated for app + * packet number space and for the packets sent after handshake + * is confirmed. During handshake, there is not much packets + * sent and also people seem to do lots of effort not to trigger + * persistent congestion there, then it is a lot easier to just + * not enable it during handshake. + */ + if (rtb->pktns_id == NGTCP2_PKTNS_ID_APP && loss_window > 0) { if (loss_window >= congestion_period) { ngtcp2_log_info(rtb->log, NGTCP2_LOG_EVENT_RCV, "persistent congestion loss_window=%" PRIu64 @@ -600,100 +905,218 @@ void ngtcp2_rtb_detect_lost_pkt(ngtcp2_rtb *rtb, ngtcp2_frame_chain **pfrc, } } - return; + break; } } + + ngtcp2_rtb_remove_excessive_lost_pkt(rtb, pkt_thres); + + return 0; } -void ngtcp2_rtb_remove_all(ngtcp2_rtb *rtb, ngtcp2_frame_chain **pfrc, - ngtcp2_conn_stat *cstat) { +void ngtcp2_rtb_remove_excessive_lost_pkt(ngtcp2_rtb *rtb, size_t n) { + ngtcp2_ksl_it it = ngtcp2_ksl_end(&rtb->ents); ngtcp2_rtb_entry *ent; - ngtcp2_ksl_it it; - - it = ngtcp2_ksl_begin(&rtb->ents); + int rv; - for (; !ngtcp2_ksl_it_end(&it);) { + for (; rtb->num_lost_pkts > n;) { + assert(ngtcp2_ksl_it_end(&it)); + ngtcp2_ksl_it_prev(&it); ent = ngtcp2_ksl_it_get(&it); - rtb_on_remove(rtb, ent, cstat); - ngtcp2_ksl_remove(&rtb->ents, &it, &ent->hd.pkt_num); + assert(ent->flags & NGTCP2_RTB_FLAG_LOST_RETRANSMITTED); - rtb_on_pkt_lost(rtb, pfrc, ent); + ngtcp2_log_info(rtb->log, NGTCP2_LOG_EVENT_RCV, + "removing stale lost pkn=%" PRId64, ent->hd.pkt_num); + + --rtb->num_lost_pkts; + rv = ngtcp2_ksl_remove(&rtb->ents, &it, &ent->hd.pkt_num); + assert(0 == rv); + ngtcp2_rtb_entry_del(ent, rtb->mem); } } -int ngtcp2_rtb_on_crypto_timeout(ngtcp2_rtb *rtb, ngtcp2_frame_chain **pfrc, - ngtcp2_conn_stat *cstat) { - ngtcp2_rtb_entry *ent; +void ngtcp2_rtb_remove_expired_lost_pkt(ngtcp2_rtb *rtb, ngtcp2_duration pto, + ngtcp2_tstamp ts) { ngtcp2_ksl_it it; - ngtcp2_frame_chain *nfrc; - ngtcp2_frame_chain *frc; - ngtcp2_ksl_it gapit; - ngtcp2_range gap, range; - ngtcp2_crypto *fr; - int all_acked; + ngtcp2_rtb_entry *ent; int rv; - it = ngtcp2_ksl_begin(&rtb->ents); + if (ngtcp2_ksl_len(&rtb->ents) == 0) { + return; + } - for (; !ngtcp2_ksl_it_end(&it);) { + it = ngtcp2_ksl_end(&rtb->ents); + + for (;;) { + assert(ngtcp2_ksl_it_end(&it)); + + ngtcp2_ksl_it_prev(&it); ent = ngtcp2_ksl_it_get(&it); - if ((ent->flags & NGTCP2_RTB_FLAG_PROBE) || - !(ent->flags & NGTCP2_RTB_FLAG_CRYPTO_PKT)) { - ngtcp2_ksl_it_next(&it); - continue; + if (!(ent->flags & NGTCP2_RTB_FLAG_LOST_RETRANSMITTED) || + ts - ent->lost_ts < pto) { + return; } - all_acked = 1; + ngtcp2_log_info(rtb->log, NGTCP2_LOG_EVENT_RCV, + "removing stale lost pkn=%" PRId64, ent->hd.pkt_num); - for (frc = ent->frc; frc; frc = frc->next) { - assert(frc->fr.type == NGTCP2_FRAME_CRYPTO); + --rtb->num_lost_pkts; + rv = ngtcp2_ksl_remove(&rtb->ents, &it, &ent->hd.pkt_num); + assert(0 == rv); + ngtcp2_rtb_entry_del(ent, rtb->mem); - fr = &frc->fr.crypto; + if (ngtcp2_ksl_len(&rtb->ents) == 0) { + return; + } + } +} - /* Don't resend CRYPTO frame if the whole region it contains has - been acknowledged */ - gapit = ngtcp2_gaptr_get_first_gap_after(&rtb->crypto->tx.acked_offset, - fr->offset); - gap = *(ngtcp2_range *)ngtcp2_ksl_it_key(&gapit); +ngtcp2_tstamp ngtcp2_rtb_lost_pkt_ts(ngtcp2_rtb *rtb) { + ngtcp2_ksl_it it; + ngtcp2_rtb_entry *ent; - range.begin = fr->offset; - range.end = fr->offset + ngtcp2_vec_len(fr->data, fr->datacnt); - range = ngtcp2_range_intersect(&range, &gap); - if (ngtcp2_range_len(&range) == 0) { - continue; - } + if (ngtcp2_ksl_len(&rtb->ents) == 0) { + return UINT64_MAX; + } + + it = ngtcp2_ksl_end(&rtb->ents); + ngtcp2_ksl_it_prev(&it); + ent = ngtcp2_ksl_it_get(&it); + + if (!(ent->flags & NGTCP2_RTB_FLAG_LOST_RETRANSMITTED)) { + return UINT64_MAX; + } + + return ent->lost_ts; +} + +static int rtb_on_pkt_lost_resched_move(ngtcp2_rtb *rtb, ngtcp2_conn *conn, + ngtcp2_pktns *pktns, + ngtcp2_rtb_entry *ent) { + ngtcp2_frame_chain **pfrc, *frc; + ngtcp2_stream *sfr; + ngtcp2_strm *strm; + int rv; + + ngtcp2_log_pkt_lost(rtb->log, ent->hd.pkt_num, ent->hd.type, ent->hd.flags, + ent->ts); + + if (rtb->qlog) { + ngtcp2_qlog_pkt_lost(rtb->qlog, ent); + } + + if (ent->flags & NGTCP2_RTB_FLAG_PROBE) { + ngtcp2_log_info(rtb->log, NGTCP2_LOG_EVENT_RCV, + "pkn=%" PRId64 + " is a probe packet, no retransmission is necessary", + ent->hd.pkt_num); + return 0; + } + + if (!ent->frc) { + /* PADDING only (or PADDING + ACK ) packets will have NULL + ent->frc. */ + assert(!(ent->flags & NGTCP2_RTB_FLAG_LOST_RETRANSMITTED)); + assert(!(ent->flags & NGTCP2_RTB_FLAG_PTO_RECLAIMED)); + return 0; + } - all_acked = 0; + if (ent->flags & NGTCP2_RTB_FLAG_LOST_RETRANSMITTED) { + --rtb->num_lost_pkts; + } - if (!(ent->flags & NGTCP2_RTB_FLAG_CRYPTO_TIMEOUT_RETRANSMITTED)) { - rv = ngtcp2_frame_chain_crypto_datacnt_new( - &nfrc, frc->fr.crypto.datacnt, rtb->mem); + if (ent->flags & NGTCP2_RTB_FLAG_LOST_RETRANSMITTED) { + ngtcp2_log_info(rtb->log, NGTCP2_LOG_EVENT_RCV, + "pkn=%" PRId64 + " was declared lost and has already been retransmitted", + ent->hd.pkt_num); + return 0; + } + + if (ent->flags & NGTCP2_RTB_FLAG_PTO_RECLAIMED) { + ngtcp2_log_info(rtb->log, NGTCP2_LOG_EVENT_RCV, + "pkn=%" PRId64 " has already been reclaimed on PTO", + ent->hd.pkt_num); + return 0; + } + + pfrc = &ent->frc; + + for (; *pfrc;) { + switch ((*pfrc)->fr.type) { + case NGTCP2_FRAME_STREAM: + frc = *pfrc; + + *pfrc = frc->next; + frc->next = NULL; + sfr = &frc->fr.stream; + + strm = ngtcp2_conn_find_stream(conn, sfr->stream_id); + if (!strm) { + ngtcp2_frame_chain_del(frc, conn->mem); + break; + } + rv = ngtcp2_strm_streamfrq_push(strm, frc); + if (rv != 0) { + ngtcp2_frame_chain_del(frc, conn->mem); + return rv; + } + if (!ngtcp2_strm_is_tx_queued(strm)) { + strm->cycle = ngtcp2_conn_tx_strmq_first_cycle(conn); + rv = ngtcp2_conn_tx_strmq_push(conn, strm); if (rv != 0) { return rv; } + } + break; + case NGTCP2_FRAME_CRYPTO: + frc = *pfrc; - nfrc->fr = frc->fr; - ngtcp2_vec_copy(nfrc->fr.crypto.data, frc->fr.crypto.data, - frc->fr.crypto.datacnt); + *pfrc = frc->next; + frc->next = NULL; - frame_chain_insert(pfrc, nfrc); + rv = ngtcp2_ksl_insert(&pktns->crypto.tx.frq, NULL, + &frc->fr.crypto.offset, frc); + if (rv != 0) { + assert(ngtcp2_err_is_fatal(rv)); + ngtcp2_frame_chain_del(frc, conn->mem); + return rv; } + break; + default: + pfrc = &(*pfrc)->next; } + } - if (all_acked) { - /* If the frames that ent contains have been acknowledged, - remove it from rtb. Otherwise crypto timer keeps firing. */ - rtb_on_remove(rtb, ent, cstat); - ngtcp2_ksl_remove(&rtb->ents, &it, &ent->hd.pkt_num); - ngtcp2_rtb_entry_del(ent, rtb->mem); - continue; - } + *pfrc = pktns->tx.frq; + pktns->tx.frq = ent->frc; + ent->frc = NULL; + + return 0; +} + +int ngtcp2_rtb_remove_all(ngtcp2_rtb *rtb, ngtcp2_conn *conn, + ngtcp2_pktns *pktns, ngtcp2_conn_stat *cstat) { + ngtcp2_rtb_entry *ent; + ngtcp2_ksl_it it; + int rv; + + it = ngtcp2_ksl_begin(&rtb->ents); + + for (; !ngtcp2_ksl_it_end(&it);) { + ent = ngtcp2_ksl_it_get(&it); - ent->flags |= NGTCP2_RTB_FLAG_CRYPTO_TIMEOUT_RETRANSMITTED; + rtb_on_remove(rtb, ent, cstat); + rv = ngtcp2_ksl_remove(&rtb->ents, &it, &ent->hd.pkt_num); + assert(0 == rv); - ngtcp2_ksl_it_next(&it); + rv = rtb_on_pkt_lost_resched_move(rtb, conn, pktns, ent); + ngtcp2_rtb_entry_del(ent, rtb->mem); + if (rv != 0) { + return rv; + } } return 0; @@ -707,3 +1130,43 @@ void ngtcp2_rtb_reset_cc_state(ngtcp2_rtb *rtb, int64_t cc_pkt_num) { rtb->cc_pkt_num = cc_pkt_num; rtb->cc_bytes_in_flight = 0; } + +ngtcp2_ssize ngtcp2_rtb_reclaim_on_pto(ngtcp2_rtb *rtb, ngtcp2_conn *conn, + ngtcp2_pktns *pktns, size_t num_pkts) { + ngtcp2_ksl_it it; + ngtcp2_rtb_entry *ent; + ngtcp2_ssize reclaimed; + size_t atmost = num_pkts; + + it = ngtcp2_ksl_end(&rtb->ents); + for (; !ngtcp2_ksl_it_begin(&it) && num_pkts >= 1;) { + ngtcp2_ksl_it_prev(&it); + ent = ngtcp2_ksl_it_get(&it); + + if ((ent->flags & (NGTCP2_RTB_FLAG_LOST_RETRANSMITTED | + NGTCP2_RTB_FLAG_PTO_RECLAIMED)) || + !(ent->flags & NGTCP2_RTB_FLAG_RETRANSMITTABLE)) { + continue; + } + + assert(ent->frc); + + reclaimed = rtb_reclaim_frame(rtb, conn, pktns, ent); + if (reclaimed < 0) { + return reclaimed; + } + + /* Mark reclaimed even if reclaimed == 0 so that we can skip it in + the next run. */ + ent->flags |= NGTCP2_RTB_FLAG_PTO_RECLAIMED; + + assert(rtb->num_retransmittable); + --rtb->num_retransmittable; + + if (reclaimed) { + --num_pkts; + } + } + + return (ngtcp2_ssize)(atmost - num_pkts); +} diff --git a/deps/ngtcp2/lib/ngtcp2_rtb.h b/deps/ngtcp2/lib/ngtcp2_rtb.h index 6dac9b6357e69b..0d0b738f396a23 100644 --- a/deps/ngtcp2/lib/ngtcp2_rtb.h +++ b/deps/ngtcp2/lib/ngtcp2_rtb.h @@ -38,6 +38,8 @@ struct ngtcp2_conn; typedef struct ngtcp2_conn ngtcp2_conn; +typedef struct ngtcp2_pktns ngtcp2_pktns; + struct ngtcp2_frame_chain; typedef struct ngtcp2_frame_chain ngtcp2_frame_chain; @@ -53,11 +55,47 @@ typedef struct ngtcp2_strm ngtcp2_strm; struct ngtcp2_rst; typedef struct ngtcp2_rst ngtcp2_rst; +typedef enum ngtcp2_frame_chain_binder_flag { + NGTCP2_FRAME_CHAIN_BINDER_FLAG_NONE = 0x00, + /* NGTCP2_FRAME_CHAIN_BINDER_FLAG_ACK indicates that an information + which a frame carries has been acknowledged. */ + NGTCP2_FRAME_CHAIN_BINDER_FLAG_ACK = 0x01, +} ngtcp2_frame_chain_binder_flag; + +/* + * ngtcp2_frame_chain_binder binds 2 or more of ngtcp2_frame_chain to + * share the acknowledgement state. In general, all + * ngtcp2_frame_chains bound to the same binder must have the same + * information. + */ +typedef struct ngtcp2_frame_chain_binder { + size_t refcount; + uint32_t flags; +} ngtcp2_frame_chain_binder; + +int ngtcp2_frame_chain_binder_new(ngtcp2_frame_chain_binder **pbinder, + const ngtcp2_mem *mem); + +/* + * ngtcp2_bind_frame_chains binds two frame chains |a| and |b| using + * new or existing ngtcp2_frame_chain_binder. |a| might have non-NULL + * a->binder. |b| must not have non-NULL b->binder. + * + * This function returns 0 if it succeeds, or one of the following + * negative error codes: + * + * NGTCP2_ERR_NOMEM + * Out of memory + */ +int ngtcp2_bind_frame_chains(ngtcp2_frame_chain *a, ngtcp2_frame_chain *b, + const ngtcp2_mem *mem); + /* * ngtcp2_frame_chain chains frames in a single packet. */ struct ngtcp2_frame_chain { ngtcp2_frame_chain *next; + ngtcp2_frame_chain_binder *binder; ngtcp2_frame fr; }; @@ -111,6 +149,10 @@ int ngtcp2_frame_chain_crypto_datacnt_new(ngtcp2_frame_chain **pfrc, size_t datacnt, const ngtcp2_mem *mem); +int ngtcp2_frame_chain_new_token_new(ngtcp2_frame_chain **pfrc, + const ngtcp2_vec *token, + const ngtcp2_mem *mem); + /* * ngtcp2_frame_chain_del deallocates |frc|. It also deallocates the * memory pointed by |frc|. @@ -134,15 +176,23 @@ typedef enum { /* NGTCP2_RTB_FLAG_PROBE indicates that the entry includes a probe packet. */ NGTCP2_RTB_FLAG_PROBE = 0x01, - /* NGTCP2_RTB_FLAG_CRYPTO_PKT indicates that the entry includes - handshake CRYPTO frame. */ - NGTCP2_RTB_FLAG_CRYPTO_PKT = 0x02, + /* NGTCP2_RTB_FLAG_RETRANSMITTABLE indicates that the entry includes + a frame which must be retransmitted until it is acknowledged. In + most cases, this flag is used along with + NGTCP2_RTB_FLAG_ACK_ELICITING. We have these 2 flags because + NGTCP2_RTB_FLAG_RETRANSMITTABLE triggers PTO, but just + NGTCP2_RTB_FLAG_ACK_ELICITING does not. */ + NGTCP2_RTB_FLAG_RETRANSMITTABLE = 0x02, /* NGTCP2_RTB_FLAG_ACK_ELICITING indicates that the entry elicits acknowledgement. */ NGTCP2_RTB_FLAG_ACK_ELICITING = 0x04, - /* NGTCP2_RTB_FLAG_CRYPTO_TIMEOUT_RETRANSMITTED indicates that the - CRYPTO frames have been retransmitted. */ - NGTCP2_RTB_FLAG_CRYPTO_TIMEOUT_RETRANSMITTED = 0x08, + /* NGTCP2_RTB_FLAG_PTO_RECLAIMED indicates that the packet has been + reclaimed on PTO. It is not marked lost yet and still consumes + congestion window. */ + NGTCP2_RTB_FLAG_PTO_RECLAIMED = 0x08, + /* NGTCP2_RTB_FLAG_LOST_RETRANSMITTED indicates that the entry has + been marked lost and scheduled to retransmit. */ + NGTCP2_RTB_FLAG_LOST_RETRANSMITTED = 0x10, } ngtcp2_rtb_flag; struct ngtcp2_rtb_entry; @@ -164,6 +214,8 @@ struct ngtcp2_rtb_entry { /* ts is the time point when a packet included in this entry is sent to a peer. */ ngtcp2_tstamp ts; + /* lost_ts is the time when this entry is marked lost. */ + ngtcp2_tstamp lost_ts; /* pktlen is the length of QUIC packet */ size_t pktlen; struct { @@ -217,6 +269,9 @@ typedef struct { int64_t largest_acked_tx_pkt_num; /* num_ack_eliciting is the number of ACK eliciting entries. */ size_t num_ack_eliciting; + /* num_retransmittable is the number of packets which contain frames + that must be retransmitted on loss. */ + size_t num_retransmittable; /* probe_pkt_left is the number of probe packet to send */ size_t probe_pkt_left; /* pktns_id is the identifier of packet number space. */ @@ -228,6 +283,13 @@ typedef struct { contributed to ngtcp2_conn_stat.bytes_in_flight. It only includes the bytes after congestion state is reset. */ uint64_t cc_bytes_in_flight; + /* persistent_congestion_start_ts is the time when persistent + congestion evaluation is started. It happens roughly after + handshake is confirmed. */ + ngtcp2_tstamp persistent_congestion_start_ts; + /* num_lost_pkts is the number entries in ents which has + NGTCP2_RTB_FLAG_LOST_RETRANSMITTED flag set. */ + size_t num_lost_pkts; } ngtcp2_rtb; /* @@ -286,31 +348,30 @@ ngtcp2_ssize ngtcp2_rtb_recv_ack(ngtcp2_rtb *rtb, const ngtcp2_ack *fr, * some frames might be prepended to |*pfrc| and the caller should * handle them. |pto| is PTO. */ -void ngtcp2_rtb_detect_lost_pkt(ngtcp2_rtb *rtb, ngtcp2_frame_chain **pfrc, - ngtcp2_conn_stat *cstat, ngtcp2_duration pto, - ngtcp2_tstamp ts); +int ngtcp2_rtb_detect_lost_pkt(ngtcp2_rtb *rtb, ngtcp2_conn *conn, + ngtcp2_pktns *pktns, ngtcp2_conn_stat *cstat, + ngtcp2_duration pto, ngtcp2_tstamp ts); /* - * ngtcp2_rtb_remove_all removes all packets from |rtb| and prepends - * all frames to |*pfrc|. Even when this function fails, some frames - * might be prepended to |*pfrc| and the caller should handle them. + * ngtcp2_rtb_remove_expired_lost_pkt removes expired lost packet. */ -void ngtcp2_rtb_remove_all(ngtcp2_rtb *rtb, ngtcp2_frame_chain **pfrc, - ngtcp2_conn_stat *cstat); +void ngtcp2_rtb_remove_expired_lost_pkt(ngtcp2_rtb *rtb, ngtcp2_duration pto, + ngtcp2_tstamp ts); /* - * ngtcp2_rtb_on_crypto_timeout copies all unacknowledged CRYPTO - * frames and links them to |*pfrc|. The affected ngtcp2_rtb_entry - * will have NGTCP2_RTB_FLAG_CRYPTO_TIMEOUT_RETRANSMITTED set. - * - * This function returns 0 if it succeeds, or one of the following - * negative error codes: - * - * NGTCP2_ERR_NOMEM - * Out of memory + * ngtcp2_rtb_lost_pkt_ts returns the earliest time when the still + * retained packet was lost. It returns UINT64_MAX if no such packet + * exists. + */ +ngtcp2_tstamp ngtcp2_rtb_lost_pkt_ts(ngtcp2_rtb *rtb); + +/* + * ngtcp2_rtb_remove_all removes all packets from |rtb| and prepends + * all frames to |*pfrc|. Even when this function fails, some frames + * might be prepended to |*pfrc| and the caller should handle them. */ -int ngtcp2_rtb_on_crypto_timeout(ngtcp2_rtb *rtb, ngtcp2_frame_chain **pfrc, - ngtcp2_conn_stat *cstat); +int ngtcp2_rtb_remove_all(ngtcp2_rtb *rtb, ngtcp2_conn *conn, + ngtcp2_pktns *pktns, ngtcp2_conn_stat *cstat); /* * ngtcp2_rtb_empty returns nonzero if |rtb| have no entry. @@ -324,4 +385,24 @@ int ngtcp2_rtb_empty(ngtcp2_rtb *rtb); */ void ngtcp2_rtb_reset_cc_state(ngtcp2_rtb *rtb, int64_t cc_pkt_num); +/* + * ngtcp2_rtb_remove_expired_lost_pkt ensures that the number of lost + * packets at most |n|. + */ +void ngtcp2_rtb_remove_excessive_lost_pkt(ngtcp2_rtb *rtb, size_t n); + +/* + * ngtcp2_rtb_reclaim_on_pto reclaims up to |num_pkts| packets which + * are in-flight and not marked lost to send them in PTO probe. The + * reclaimed frames are chained to |*pfrc|. + * + * This function returns the number of packets reclaimed if it + * succeeds, or one of the following negative error codes: + * + * NGTCP2_ERR_NOMEM + * Out of memory + */ +ngtcp2_ssize ngtcp2_rtb_reclaim_on_pto(ngtcp2_rtb *rtb, ngtcp2_conn *conn, + ngtcp2_pktns *pktns, size_t num_pkts); + #endif /* NGTCP2_RTB_H */ diff --git a/deps/ngtcp2/lib/ngtcp2_strm.c b/deps/ngtcp2/lib/ngtcp2_strm.c index 6c02e8a2274bdc..6fb73dc0727b5f 100644 --- a/deps/ngtcp2/lib/ngtcp2_strm.c +++ b/deps/ngtcp2/lib/ngtcp2_strm.c @@ -38,11 +38,14 @@ static int offset_less(const ngtcp2_ksl_key *lhs, const ngtcp2_ksl_key *rhs) { int ngtcp2_strm_init(ngtcp2_strm *strm, int64_t stream_id, uint32_t flags, uint64_t max_rx_offset, uint64_t max_tx_offset, void *stream_user_data, const ngtcp2_mem *mem) { - int rv; - strm->cycle = 0; + strm->tx.acked_offset = NULL; + strm->tx.cont_acked_offset = 0; + strm->tx.streamfrq = NULL; strm->tx.offset = 0; strm->tx.max_offset = max_tx_offset; + strm->rx.rob = NULL; + strm->rx.cont_offset = 0; strm->rx.last_offset = 0; strm->stream_id = stream_id; strm->flags = flags; @@ -54,29 +57,7 @@ int ngtcp2_strm_init(ngtcp2_strm *strm, int64_t stream_id, uint32_t flags, strm->mem = mem; strm->app_error_code = 0; - rv = ngtcp2_gaptr_init(&strm->tx.acked_offset, mem); - if (rv != 0) { - goto fail_gaptr_init; - } - - rv = ngtcp2_rob_init(&strm->rx.rob, 8 * 1024, mem); - if (rv != 0) { - goto fail_rob_init; - } - - rv = ngtcp2_ksl_init(&strm->tx.streamfrq, offset_less, sizeof(uint64_t), mem); - if (rv != 0) { - goto fail_tx_streamfrq_init; - } - return 0; - -fail_tx_streamfrq_init: - ngtcp2_rob_free(&strm->rx.rob); -fail_rob_init: - ngtcp2_gaptr_free(&strm->tx.acked_offset); -fail_gaptr_init: - return rv; } void ngtcp2_strm_free(ngtcp2_strm *strm) { @@ -86,37 +67,268 @@ void ngtcp2_strm_free(ngtcp2_strm *strm) { return; } - for (it = ngtcp2_ksl_begin(&strm->tx.streamfrq); !ngtcp2_ksl_it_end(&it); - ngtcp2_ksl_it_next(&it)) { - ngtcp2_frame_chain_del(ngtcp2_ksl_it_get(&it), strm->mem); + if (strm->tx.streamfrq) { + for (it = ngtcp2_ksl_begin(strm->tx.streamfrq); !ngtcp2_ksl_it_end(&it); + ngtcp2_ksl_it_next(&it)) { + ngtcp2_frame_chain_del(ngtcp2_ksl_it_get(&it), strm->mem); + } + + ngtcp2_ksl_free(strm->tx.streamfrq); + ngtcp2_mem_free(strm->mem, strm->tx.streamfrq); + } + + ngtcp2_rob_free(strm->rx.rob); + ngtcp2_mem_free(strm->mem, strm->rx.rob); + ngtcp2_gaptr_free(strm->tx.acked_offset); + ngtcp2_mem_free(strm->mem, strm->tx.acked_offset); +} + +static int strm_rob_init(ngtcp2_strm *strm) { + int rv; + ngtcp2_rob *rob = ngtcp2_mem_malloc(strm->mem, sizeof(*rob)); + + if (rob == NULL) { + return NGTCP2_ERR_NOMEM; } - ngtcp2_ksl_free(&strm->tx.streamfrq); - ngtcp2_rob_free(&strm->rx.rob); - ngtcp2_gaptr_free(&strm->tx.acked_offset); + rv = ngtcp2_rob_init(rob, 8 * 1024, strm->mem); + if (rv != 0) { + ngtcp2_mem_free(strm->mem, rob); + return rv; + } + + strm->rx.rob = rob; + + return 0; } uint64_t ngtcp2_strm_rx_offset(ngtcp2_strm *strm) { - return ngtcp2_rob_first_gap_offset(&strm->rx.rob); + if (strm->rx.rob == NULL) { + return strm->rx.cont_offset; + } + return ngtcp2_rob_first_gap_offset(strm->rx.rob); } int ngtcp2_strm_recv_reordering(ngtcp2_strm *strm, const uint8_t *data, size_t datalen, uint64_t offset) { - return ngtcp2_rob_push(&strm->rx.rob, offset, data, datalen); + int rv; + + if (strm->rx.rob == NULL) { + rv = strm_rob_init(strm); + if (rv != 0) { + return rv; + } + + if (strm->rx.cont_offset) { + rv = ngtcp2_rob_remove_prefix(strm->rx.rob, strm->rx.cont_offset); + if (rv != 0) { + return rv; + } + } + } + + return ngtcp2_rob_push(strm->rx.rob, offset, data, datalen); +} + +int ngtcp2_strm_update_rx_offset(ngtcp2_strm *strm, uint64_t offset) { + if (strm->rx.rob == NULL) { + strm->rx.cont_offset = offset; + return 0; + } + + return ngtcp2_rob_remove_prefix(strm->rx.rob, offset); } void ngtcp2_strm_shutdown(ngtcp2_strm *strm, uint32_t flags) { strm->flags |= flags & NGTCP2_STRM_FLAG_SHUT_RDWR; } +static int strm_streamfrq_init(ngtcp2_strm *strm) { + int rv; + ngtcp2_ksl *streamfrq = ngtcp2_mem_malloc(strm->mem, sizeof(*streamfrq)); + if (streamfrq == NULL) { + return NGTCP2_ERR_NOMEM; + } + + rv = ngtcp2_ksl_init(streamfrq, offset_less, sizeof(uint64_t), strm->mem); + if (rv != 0) { + ngtcp2_mem_free(strm->mem, streamfrq); + return rv; + } + + strm->tx.streamfrq = streamfrq; + + return 0; +} + int ngtcp2_strm_streamfrq_push(ngtcp2_strm *strm, ngtcp2_frame_chain *frc) { + int rv; + assert(frc->fr.type == NGTCP2_FRAME_STREAM); assert(frc->next == NULL); - return ngtcp2_ksl_insert(&strm->tx.streamfrq, NULL, &frc->fr.stream.offset, + if (strm->tx.streamfrq == NULL) { + rv = strm_streamfrq_init(strm); + if (rv != 0) { + return rv; + } + } + + return ngtcp2_ksl_insert(strm->tx.streamfrq, NULL, &frc->fr.stream.offset, frc); } +static int strm_streamfrq_unacked_pop(ngtcp2_strm *strm, + ngtcp2_frame_chain **pfrc) { + ngtcp2_frame_chain *frc, *nfrc; + ngtcp2_stream *fr, *nfr; + uint64_t offset, end_offset; + size_t idx, end_idx; + uint64_t base_offset, end_base_offset; + ngtcp2_range gap; + ngtcp2_vec *v; + int rv; + ngtcp2_ksl_it it; + + *pfrc = NULL; + + assert(strm->tx.streamfrq); + assert(ngtcp2_ksl_len(strm->tx.streamfrq)); + + for (it = ngtcp2_ksl_begin(strm->tx.streamfrq); !ngtcp2_ksl_it_end(&it);) { + frc = ngtcp2_ksl_it_get(&it); + fr = &frc->fr.stream; + + ngtcp2_ksl_remove(strm->tx.streamfrq, &it, &fr->offset); + + idx = 0; + offset = fr->offset; + base_offset = 0; + + gap = ngtcp2_strm_get_unacked_range_after(strm, offset); + if (gap.begin < offset) { + gap.begin = offset; + } + + for (; idx < fr->datacnt && offset < gap.begin; ++idx) { + v = &fr->data[idx]; + if (offset + v->len > gap.begin) { + base_offset = gap.begin - offset; + break; + } + + offset += v->len; + } + + if (idx == fr->datacnt) { + if (fr->fin) { + if (strm->flags & NGTCP2_STRM_FLAG_FIN_ACKED) { + ngtcp2_frame_chain_del(frc, strm->mem); + assert(ngtcp2_ksl_len(strm->tx.streamfrq) == 0); + return 0; + } + + fr->offset = fr->offset + ngtcp2_vec_len(fr->data, fr->datacnt); + fr->datacnt = 0; + + *pfrc = frc; + + return 0; + } + ngtcp2_frame_chain_del(frc, strm->mem); + continue; + } + + assert(gap.begin == offset + base_offset); + + end_idx = idx; + end_offset = offset; + end_base_offset = 0; + + for (; end_idx < fr->datacnt; ++end_idx) { + v = &fr->data[end_idx]; + if (end_offset + v->len > gap.end) { + end_base_offset = gap.end - end_offset; + break; + } + + end_offset += v->len; + } + + if (fr->offset == offset && base_offset == 0 && fr->datacnt == end_idx) { + *pfrc = frc; + return 0; + } + + if (fr->datacnt == end_idx) { + memmove(fr->data, fr->data + idx, sizeof(fr->data[0]) * (end_idx - idx)); + + assert(fr->data[0].len > base_offset); + + fr->offset = offset + base_offset; + fr->datacnt = end_idx - idx; + fr->data[0].base += base_offset; + fr->data[0].len -= (size_t)base_offset; + + *pfrc = frc; + return 0; + } + + rv = ngtcp2_frame_chain_stream_datacnt_new(&nfrc, fr->datacnt - end_idx, + strm->mem); + if (rv != 0) { + ngtcp2_frame_chain_del(frc, strm->mem); + return rv; + } + + nfr = &nfrc->fr.stream; + memcpy(nfr->data, fr->data + end_idx, + sizeof(nfr->data[0]) * (fr->datacnt - end_idx)); + + assert(nfr->data[0].len > end_base_offset); + + nfr->type = NGTCP2_FRAME_STREAM; + nfr->flags = 0; + nfr->fin = fr->fin; + nfr->stream_id = fr->stream_id; + nfr->offset = end_offset + end_base_offset; + nfr->datacnt = fr->datacnt - end_idx; + nfr->data[0].base += end_base_offset; + nfr->data[0].len -= (size_t)end_base_offset; + + rv = ngtcp2_ksl_insert(strm->tx.streamfrq, NULL, &nfr->offset, nfrc); + if (rv != 0) { + assert(ngtcp2_err_is_fatal(rv)); + ngtcp2_frame_chain_del(nfrc, strm->mem); + ngtcp2_frame_chain_del(frc, strm->mem); + return rv; + } + + if (end_base_offset) { + ++end_idx; + } + + memmove(fr->data, fr->data + idx, sizeof(fr->data[0]) * (end_idx - idx)); + + assert(fr->data[0].len > base_offset); + + fr->fin = 0; + fr->offset = offset + base_offset; + fr->datacnt = end_idx - idx; + if (end_base_offset) { + assert(fr->data[fr->datacnt - 1].len > end_base_offset); + fr->data[fr->datacnt - 1].len = (size_t)end_base_offset; + } + fr->data[0].base += base_offset; + fr->data[0].len -= (size_t)base_offset; + + *pfrc = frc; + return 0; + } + + return 0; +} + int ngtcp2_strm_streamfrq_pop(ngtcp2_strm *strm, ngtcp2_frame_chain **pfrc, size_t left) { ngtcp2_stream *fr, *nfr; @@ -127,30 +339,39 @@ int ngtcp2_strm_streamfrq_pop(ngtcp2_strm *strm, ngtcp2_frame_chain **pfrc, ngtcp2_vec a[NGTCP2_MAX_STREAM_DATACNT]; ngtcp2_vec b[NGTCP2_MAX_STREAM_DATACNT]; size_t acnt, bcnt; - ngtcp2_ksl_it it; - uint64_t old_offset; + uint64_t unacked_offset; - if (ngtcp2_ksl_len(&strm->tx.streamfrq) == 0) { + if (strm->tx.streamfrq == NULL || ngtcp2_ksl_len(strm->tx.streamfrq) == 0) { *pfrc = NULL; return 0; } - it = ngtcp2_ksl_begin(&strm->tx.streamfrq); - frc = ngtcp2_ksl_it_get(&it); - fr = &frc->fr.stream; + rv = strm_streamfrq_unacked_pop(strm, &frc); + if (rv != 0) { + return rv; + } + if (frc == NULL) { + *pfrc = NULL; + return 0; + } + fr = &frc->fr.stream; datalen = ngtcp2_vec_len(fr->data, fr->datacnt); if (left == 0) { /* datalen could be zero if 0 length STREAM has been sent */ - if (datalen || ngtcp2_ksl_len(&strm->tx.streamfrq) > 1) { + if (datalen || ngtcp2_ksl_len(strm->tx.streamfrq) > 1) { + rv = ngtcp2_ksl_insert(strm->tx.streamfrq, NULL, &fr->offset, frc); + if (rv != 0) { + assert(ngtcp2_err_is_fatal(rv)); + ngtcp2_frame_chain_del(frc, strm->mem); + return rv; + } *pfrc = NULL; return 0; } } - ngtcp2_ksl_remove(&strm->tx.streamfrq, NULL, &fr->offset); - if (datalen > left) { ngtcp2_vec_copy(a, fr->data, fr->datacnt); acnt = fr->datacnt; @@ -177,7 +398,7 @@ int ngtcp2_strm_streamfrq_pop(ngtcp2_strm *strm, ngtcp2_frame_chain **pfrc, nfr->datacnt = bcnt; ngtcp2_vec_copy(nfr->data, b, bcnt); - rv = ngtcp2_ksl_insert(&strm->tx.streamfrq, NULL, &nfr->offset, nfrc); + rv = ngtcp2_ksl_insert(strm->tx.streamfrq, NULL, &nfr->offset, nfrc); if (rv != 0) { assert(ngtcp2_err_is_fatal(rv)); ngtcp2_frame_chain_del(nfrc, strm->mem); @@ -210,19 +431,27 @@ int ngtcp2_strm_streamfrq_pop(ngtcp2_strm *strm, ngtcp2_frame_chain **pfrc, ngtcp2_vec_copy(a, fr->data, fr->datacnt); acnt = fr->datacnt; - for (; left && ngtcp2_ksl_len(&strm->tx.streamfrq);) { - it = ngtcp2_ksl_begin(&strm->tx.streamfrq); - nfrc = ngtcp2_ksl_it_get(&it); - nfr = &nfrc->fr.stream; + for (; left && ngtcp2_ksl_len(strm->tx.streamfrq);) { + unacked_offset = ngtcp2_strm_streamfrq_unacked_offset(strm); + if (unacked_offset != fr->offset + datalen) { + assert(fr->offset + datalen < unacked_offset); + break; + } - if (nfr->offset != fr->offset + datalen) { - assert(fr->offset + datalen < nfr->offset); + rv = strm_streamfrq_unacked_pop(strm, &nfrc); + if (rv != 0) { + assert(ngtcp2_err_is_fatal(rv)); + ngtcp2_frame_chain_del(frc, strm->mem); + return rv; + } + if (nfrc == NULL) { break; } + nfr = &nfrc->fr.stream; + if (nfr->fin && nfr->datacnt == 0) { fr->fin = 1; - ngtcp2_ksl_remove(&strm->tx.streamfrq, NULL, &nfr->offset); ngtcp2_frame_chain_del(nfrc, strm->mem); break; } @@ -230,6 +459,13 @@ int ngtcp2_strm_streamfrq_pop(ngtcp2_strm *strm, ngtcp2_frame_chain **pfrc, nmerged = ngtcp2_vec_merge(a, &acnt, nfr->data, &nfr->datacnt, left, NGTCP2_MAX_STREAM_DATACNT); if (nmerged == 0) { + rv = ngtcp2_ksl_insert(strm->tx.streamfrq, NULL, &nfr->offset, nfrc); + if (rv != 0) { + assert(ngtcp2_err_is_fatal(rv)); + ngtcp2_frame_chain_del(nfrc, strm->mem); + ngtcp2_frame_chain_del(frc, strm->mem); + return rv; + } break; } @@ -238,15 +474,18 @@ int ngtcp2_strm_streamfrq_pop(ngtcp2_strm *strm, ngtcp2_frame_chain **pfrc, if (nfr->datacnt == 0) { fr->fin = nfr->fin; - ngtcp2_ksl_remove(&strm->tx.streamfrq, NULL, &nfr->offset); ngtcp2_frame_chain_del(nfrc, strm->mem); continue; } - old_offset = nfr->offset; nfr->offset += nmerged; - ngtcp2_ksl_update_key(&strm->tx.streamfrq, &old_offset, &nfr->offset); + rv = ngtcp2_ksl_insert(strm->tx.streamfrq, NULL, &nfr->offset, nfrc); + if (rv != 0) { + ngtcp2_frame_chain_del(nfrc, strm->mem); + ngtcp2_frame_chain_del(frc, strm->mem); + return rv; + } break; } @@ -280,29 +519,68 @@ int ngtcp2_strm_streamfrq_pop(ngtcp2_strm *strm, ngtcp2_frame_chain **pfrc, return 0; } +uint64_t ngtcp2_strm_streamfrq_unacked_offset(ngtcp2_strm *strm) { + ngtcp2_frame_chain *frc; + ngtcp2_stream *fr; + ngtcp2_range gap; + ngtcp2_ksl_it it; + size_t datalen; + + assert(strm->tx.streamfrq); + assert(ngtcp2_ksl_len(strm->tx.streamfrq)); + + for (it = ngtcp2_ksl_begin(strm->tx.streamfrq); !ngtcp2_ksl_it_end(&it); + ngtcp2_ksl_it_next(&it)) { + frc = ngtcp2_ksl_it_get(&it); + fr = &frc->fr.stream; + + gap = ngtcp2_strm_get_unacked_range_after(strm, fr->offset); + + datalen = ngtcp2_vec_len(fr->data, fr->datacnt); + + if (gap.begin <= fr->offset) { + return fr->offset; + } + if (gap.begin < fr->offset + datalen) { + return gap.begin; + } + if (fr->offset + datalen == gap.begin && fr->fin && + !(strm->flags & NGTCP2_STRM_FLAG_FIN_ACKED)) { + return fr->offset + datalen; + } + } + + return (uint64_t)-1; +} + ngtcp2_frame_chain *ngtcp2_strm_streamfrq_top(ngtcp2_strm *strm) { ngtcp2_ksl_it it; - assert(ngtcp2_ksl_len(&strm->tx.streamfrq)); + assert(strm->tx.streamfrq); + assert(ngtcp2_ksl_len(strm->tx.streamfrq)); - it = ngtcp2_ksl_begin(&strm->tx.streamfrq); + it = ngtcp2_ksl_begin(strm->tx.streamfrq); return ngtcp2_ksl_it_get(&it); } int ngtcp2_strm_streamfrq_empty(ngtcp2_strm *strm) { - return ngtcp2_ksl_len(&strm->tx.streamfrq) == 0; + return strm->tx.streamfrq == NULL || ngtcp2_ksl_len(strm->tx.streamfrq) == 0; } void ngtcp2_strm_streamfrq_clear(ngtcp2_strm *strm) { ngtcp2_frame_chain *frc; ngtcp2_ksl_it it; - for (it = ngtcp2_ksl_begin(&strm->tx.streamfrq); !ngtcp2_ksl_it_end(&it); + if (strm->tx.streamfrq == NULL) { + return; + } + + for (it = ngtcp2_ksl_begin(strm->tx.streamfrq); !ngtcp2_ksl_it_end(&it); ngtcp2_ksl_it_next(&it)) { frc = ngtcp2_ksl_it_get(&it); ngtcp2_frame_chain_del(frc, strm->mem); } - ngtcp2_ksl_clear(&strm->tx.streamfrq); + ngtcp2_ksl_clear(strm->tx.streamfrq); } int ngtcp2_strm_is_tx_queued(ngtcp2_strm *strm) { @@ -310,6 +588,77 @@ int ngtcp2_strm_is_tx_queued(ngtcp2_strm *strm) { } int ngtcp2_strm_is_all_tx_data_acked(ngtcp2_strm *strm) { - return ngtcp2_gaptr_first_gap_offset(&strm->tx.acked_offset) == + if (strm->tx.acked_offset == NULL) { + return strm->tx.cont_acked_offset == strm->tx.offset; + } + + return ngtcp2_gaptr_first_gap_offset(strm->tx.acked_offset) == strm->tx.offset; } + +ngtcp2_range ngtcp2_strm_get_unacked_range_after(ngtcp2_strm *strm, + uint64_t offset) { + ngtcp2_ksl_it gapit; + ngtcp2_range gap; + + if (strm->tx.acked_offset == NULL) { + gap.begin = strm->tx.cont_acked_offset; + gap.end = UINT64_MAX; + return gap; + } + + gapit = ngtcp2_gaptr_get_first_gap_after(strm->tx.acked_offset, offset); + return *(ngtcp2_range *)ngtcp2_ksl_it_key(&gapit); +} + +uint64_t ngtcp2_strm_get_acked_offset(ngtcp2_strm *strm) { + if (strm->tx.acked_offset == NULL) { + return strm->tx.cont_acked_offset; + } + + return ngtcp2_gaptr_first_gap_offset(strm->tx.acked_offset); +} + +static int strm_acked_offset_init(ngtcp2_strm *strm) { + int rv; + ngtcp2_gaptr *acked_offset = + ngtcp2_mem_malloc(strm->mem, sizeof(*acked_offset)); + + if (acked_offset == NULL) { + return NGTCP2_ERR_NOMEM; + } + + rv = ngtcp2_gaptr_init(acked_offset, strm->mem); + if (rv != 0) { + ngtcp2_mem_free(strm->mem, acked_offset); + return rv; + } + + strm->tx.acked_offset = acked_offset; + + return 0; +} + +int ngtcp2_strm_ack_data(ngtcp2_strm *strm, uint64_t offset, uint64_t len) { + int rv; + + if (strm->tx.acked_offset == NULL) { + if (strm->tx.cont_acked_offset == offset) { + strm->tx.cont_acked_offset += len; + return 0; + } + + rv = strm_acked_offset_init(strm); + if (rv != 0) { + return rv; + } + + rv = + ngtcp2_gaptr_push(strm->tx.acked_offset, 0, strm->tx.cont_acked_offset); + if (rv != 0) { + return rv; + } + } + + return ngtcp2_gaptr_push(strm->tx.acked_offset, offset, len); +} diff --git a/deps/ngtcp2/lib/ngtcp2_strm.h b/deps/ngtcp2/lib/ngtcp2_strm.h index 67d49e519e0eda..7da8437e6681f7 100644 --- a/deps/ngtcp2/lib/ngtcp2_strm.h +++ b/deps/ngtcp2/lib/ngtcp2_strm.h @@ -64,6 +64,9 @@ typedef enum { /* NGTCP2_STRM_FLAG_RST_ACKED indicates that the outgoing RST_STREAM is acknowledged by peer. */ NGTCP2_STRM_FLAG_RST_ACKED = 0x20, + /* NGTCP2_STRM_FLAG_FIN_ACKED indicates that a STREAM with FIN bit + set is acknowledged by a remote endpoint. */ + NGTCP2_STRM_FLAG_FIN_ACKED = 0x40, } ngtcp2_strm_flags; struct ngtcp2_strm; @@ -76,12 +79,17 @@ struct ngtcp2_strm { struct { /* acked_offset tracks acknowledged outgoing data. */ - ngtcp2_gaptr acked_offset; + ngtcp2_gaptr *acked_offset; + /* cont_acked_offset is the offset that all data up to this offset + is acknowledged by a remote endpoint. It is used until the + remote endpoint acknowledges data in out-of-order. After that, + acked_offset is used instead. */ + uint64_t cont_acked_offset; /* streamfrq contains STREAM frame for retransmission. The flow control credits have been paid when they are transmitted first time. There are no restriction regarding flow control for retransmission. */ - ngtcp2_ksl streamfrq; + ngtcp2_ksl *streamfrq; /* offset is the next offset of outgoing data. In other words, it is the number of bytes sent in this stream without duplication. */ @@ -95,7 +103,11 @@ struct ngtcp2_strm { /* rob is the reorder buffer for incoming stream data. The data received in out of order is buffered and sorted by its offset in this object. */ - ngtcp2_rob rob; + ngtcp2_rob *rob; + /* cont_offset is the largest offset of consecutive data. It is + used until the endpoint receives out-of-order data. After + that, rob is used to track the offset and data. */ + uint64_t cont_offset; /* last_offset is the largest offset of stream data received for this stream. */ uint64_t last_offset; @@ -155,6 +167,15 @@ uint64_t ngtcp2_strm_rx_offset(ngtcp2_strm *strm); int ngtcp2_strm_recv_reordering(ngtcp2_strm *strm, const uint8_t *data, size_t datalen, uint64_t offset); +/* + * ngtcp2_strm_update_rx_offset tells that data up to offset bytes are + * received in order. + * + * NGTCP2_ERR_NOMEM + * Out of memory + */ +int ngtcp2_strm_update_rx_offset(ngtcp2_strm *strm, uint64_t offset); + /* * ngtcp2_strm_shutdown shutdowns |strm|. |flags| should be * NGTCP2_STRM_FLAG_SHUT_RD, and/or NGTCP2_STRM_FLAG_SHUT_WR. @@ -189,6 +210,12 @@ int ngtcp2_strm_streamfrq_push(ngtcp2_strm *strm, ngtcp2_frame_chain *frc); int ngtcp2_strm_streamfrq_pop(ngtcp2_strm *strm, ngtcp2_frame_chain **pfrc, size_t left); +/* + * ngtcp2_strm_streamfrq_unacked_offset returns the smallest offset of + * unacknowledged stream data held in strm->tx.streamfrq. + */ +uint64_t ngtcp2_strm_streamfrq_unacked_offset(ngtcp2_strm *strm); + /* * ngtcp2_strm_streamfrq_top returns the first ngtcp2_frame_chain. * The queue must not be empty. @@ -216,4 +243,24 @@ int ngtcp2_strm_is_tx_queued(ngtcp2_strm *strm); */ int ngtcp2_strm_is_all_tx_data_acked(ngtcp2_strm *strm); +/* + * ngtcp2_strm_get_unacked_range_after returns the range that is not + * acknowledged yet and intersects or comes after |offset|. + */ +ngtcp2_range ngtcp2_strm_get_unacked_range_after(ngtcp2_strm *strm, + uint64_t offset); + +/* + * ngtcp2_strm_get_acked_offset returns offset, that is the data up to + * this offset have been acknowledged by a remote endpoint. It + * returns 0 if no data is acknowledged. + */ +uint64_t ngtcp2_strm_get_acked_offset(ngtcp2_strm *strm); + +/* + * ngtcp2_strm_ack_data tells |strm| that the data [offset, + * offset+len) is acknowledged by a remote endpoint. + */ +int ngtcp2_strm_ack_data(ngtcp2_strm *strm, uint64_t offset, uint64_t len); + #endif /* NGTCP2_STRM_H */ diff --git a/src/node_sockaddr-inl.h b/src/node_sockaddr-inl.h index e5d8985771ebf2..b5ae5acb5fd7fd 100644 --- a/src/node_sockaddr-inl.h +++ b/src/node_sockaddr-inl.h @@ -152,6 +152,11 @@ void SocketAddress::Update(uint8_t* data, size_t len) { memcpy(&address_, data, len); } +void SocketAddress::Update(const sockaddr* data, size_t len) { + CHECK_LE(len, sizeof(address_)); + memcpy(&address_, data, len); +} + v8::Local SocketAddress::ToJS( Environment* env, v8::Local info) const { diff --git a/src/node_sockaddr.h b/src/node_sockaddr.h index 69a370afa32e81..f05b58d3551f00 100644 --- a/src/node_sockaddr.h +++ b/src/node_sockaddr.h @@ -123,6 +123,7 @@ class SocketAddress : public MemoryRetainer { inline void set_flow_label(uint32_t label = 0); inline void Update(uint8_t* data, size_t len); + inline void Update(const sockaddr* data, size_t len); static SocketAddress FromSockName(const uv_udp_t& handle); static SocketAddress FromSockName(const uv_tcp_t& handle); diff --git a/src/quic/node_quic_crypto.cc b/src/quic/node_quic_crypto.cc index b073ca9c381487..11a7deccb4f003 100644 --- a/src/quic/node_quic_crypto.cc +++ b/src/quic/node_quic_crypto.cc @@ -164,13 +164,22 @@ bool GenerateRetryToken( return false; } + ngtcp2_crypto_aead_ctx aead_ctx; + if (NGTCP2_ERR(ngtcp2_crypto_aead_ctx_encrypt_init( + &aead_ctx, + &ctx.aead, + token_key, + ivlen))) { + return false; + } + size_t plaintextlen = std::distance(std::begin(plaintext), p); if (NGTCP2_ERR(ngtcp2_crypto_encrypt( token, &ctx.aead, + &aead_ctx, plaintext.data(), plaintextlen, - token_key, token_iv, ivlen, addr.raw(), @@ -291,12 +300,21 @@ bool InvalidRetryToken( uint8_t plaintext[4096]; + ngtcp2_crypto_aead_ctx aead_ctx; + if (NGTCP2_ERR(ngtcp2_crypto_aead_ctx_decrypt_init( + &aead_ctx, + &ctx.aead, + token_key, + ivlen))) { + return true; + } + if (NGTCP2_ERR(ngtcp2_crypto_decrypt( plaintext, &ctx.aead, + &aead_ctx, ciphertext, ciphertextlen, - token_key, token_iv, ivlen, addr.raw(), diff --git a/src/quic/node_quic_session-inl.h b/src/quic/node_quic_session-inl.h index 8b1ac00806c4db..29cf689be7b8ef 100644 --- a/src/quic/node_quic_session-inl.h +++ b/src/quic/node_quic_session-inl.h @@ -50,6 +50,8 @@ void QuicSessionConfig::set_original_connection_id( transport_params.original_dcid = *ocid; transport_params.retry_scid = *scid; transport_params.retry_scid_present = 1; + } else { + transport_params.original_dcid = *scid; } } diff --git a/src/quic/node_quic_session.cc b/src/quic/node_quic_session.cc index 59f80c77057598..9b74f03b0cc635 100644 --- a/src/quic/node_quic_session.cc +++ b/src/quic/node_quic_session.cc @@ -1422,7 +1422,7 @@ bool QuicApplication::SendPendingData() { continue; case NGTCP2_ERR_STREAM_NOT_FOUND: continue; - case NGTCP2_ERR_WRITE_STREAM_MORE: + case NGTCP2_ERR_WRITE_MORE: CHECK_GT(ndatalen, 0); CHECK(StreamCommit(&stream_data, ndatalen)); pos += ndatalen; @@ -2096,7 +2096,6 @@ bool QuicSession::Receive( if (!is_destroyed()) UpdateIdleTimer(); - SendPendingData(); Debug(this, "Successfully processed received packet"); return true; @@ -2239,8 +2238,11 @@ void QuicSession::RemoveStream(int64_t stream_id) { void QuicSession::ScheduleRetransmit() { uint64_t now = uv_hrtime(); uint64_t expiry = ngtcp2_conn_get_expiry(connection()); - uint64_t interval = (expiry - now) / 1000000UL; - if (expiry < now || interval == 0) interval = 1; + // now and expiry are in nanoseconds, interval is milliseconds + uint64_t interval = (expiry < now) ? 1 : (expiry - now) / 1000000UL; + // If interval ends up being 0, the repeating timer won't be + // scheduled, so set it to 1 instead. + if (interval == 0) interval = 1; Debug(this, "Scheduling the retransmit timer for %" PRIu64, interval); UpdateRetransmitTimer(interval); } @@ -2440,7 +2442,7 @@ bool QuicSession::SendPacket(std::unique_ptr packet) { IncrementStat(&QuicSessionStats::bytes_sent, packet->length()); RecordTimestamp(&QuicSessionStats::sent_at); - ScheduleRetransmit(); +// ScheduleRetransmit(); Debug(this, "Sending %" PRIu64 " bytes to %s from %s", packet->length(), @@ -2471,6 +2473,7 @@ void QuicSession::SendPendingData() { Debug(this, "Error sending QUIC application data"); HandleError(); } + ScheduleRetransmit(); } // When completing the TLS handshake, the TLS session information @@ -3392,11 +3395,9 @@ int QuicSession::OnStreamReset( // Currently, there is only one use. In the future, we'll want to // explore whether we want to handle the different cases uses. int QuicSession::OnRand( - ngtcp2_conn* conn, uint8_t* dest, size_t destlen, - ngtcp2_rand_ctx ctx, - void* user_data) { + ngtcp2_rand_ctx ctx) { EntropySource(dest, destlen); return 0; } @@ -3541,6 +3542,8 @@ const ngtcp2_conn_callbacks QuicSession::callbacks[2] = { OnConnectionIDStatus, OnHandshakeConfirmed, nullptr, // recv_new_token + ngtcp2_crypto_delete_crypto_aead_ctx_cb, + ngtcp2_crypto_delete_crypto_cipher_ctx_cb, }, // NGTCP2_CRYPTO_SIDE_SERVER { @@ -3574,6 +3577,8 @@ const ngtcp2_conn_callbacks QuicSession::callbacks[2] = { OnConnectionIDStatus, nullptr, // handshake_confirmed nullptr, // recv_new_token + ngtcp2_crypto_delete_crypto_aead_ctx_cb, + ngtcp2_crypto_delete_crypto_cipher_ctx_cb, } }; @@ -3585,7 +3590,11 @@ BaseObjectPtr QuicSession::qlog_stream() { return qlog_stream_; } -void QuicSession::OnQlogWrite(void* user_data, const void* data, size_t len) { +void QuicSession::OnQlogWrite( + void* user_data, + uint32_t flags, + const void* data, + size_t len) { QuicSession* session = static_cast(user_data); Environment* env = session->env(); @@ -3599,8 +3608,9 @@ void QuicSession::OnQlogWrite(void* user_data, const void* data, size_t len) { std::vector buffer(len); memcpy(buffer.data(), data, len); env->SetImmediate([ptr = std::move(ptr), - buffer = std::move(buffer)](Environment*) { - ptr->Emit(buffer.data(), buffer.size()); + buffer = std::move(buffer), + flags](Environment*) { + ptr->Emit(buffer.data(), buffer.size(), flags); }); } @@ -3638,7 +3648,7 @@ QLogStream::QLogStream(Environment* env, v8::Local obj) StreamBase::AttachToObject(GetObject()); } -void QLogStream::Emit(const uint8_t* data, size_t len) { +void QLogStream::Emit(const uint8_t* data, size_t len, uint32_t flags) { size_t remaining = len; while (remaining != 0) { uv_buf_t buf = EmitAlloc(len); @@ -3649,10 +3659,7 @@ void QLogStream::Emit(const uint8_t* data, size_t len) { EmitRead(avail, buf); } - // The last chunk that ngtcp2 writes is 6 bytes. Unfortunately, - // this is the only way for us to know that ngtcp2 is definitely - // done sending qlog events. - if (ended_ && len == 6) + if (ended_ && flags & NGTCP2_QLOG_WRITE_FLAG_FIN) EmitRead(UV_EOF); } @@ -3888,7 +3895,6 @@ void NewQuicClientSession(const FunctionCallbackInfo& args) { args[ARG_IDX::QLOG]->IsTrue() ? QlogMode::kEnabled : QlogMode::kDisabled); - session->SendPendingData(); if (session->is_destroyed()) return args.GetReturnValue().Set(ERR_FAILED_TO_CREATE_SESSION); diff --git a/src/quic/node_quic_session.h b/src/quic/node_quic_session.h index 4bd14310e15181..7c9b3f9710f17b 100644 --- a/src/quic/node_quic_session.h +++ b/src/quic/node_quic_session.h @@ -75,6 +75,7 @@ class QuicSessionConfig final : public ngtcp2_settings { QuicSessionConfig(const QuicSessionConfig& config) { initial_ts = uv_hrtime(); + initial_rtt = config.initial_rtt; transport_params = config.transport_params; max_udp_payload_size = config.max_udp_payload_size; cc_algo = config.cc_algo; @@ -237,7 +238,7 @@ class QLogStream final : public AsyncWrap, QLogStream(Environment* env, v8::Local obj); - void Emit(const uint8_t* data, size_t len); + void Emit(const uint8_t* data, size_t len, uint32_t flags); void End() { ended_ = true; } @@ -1366,11 +1367,9 @@ class QuicSession final : public AsyncWrap, void* stream_user_data); static int OnRand( - ngtcp2_conn* conn, uint8_t* dest, size_t destlen, - ngtcp2_rand_ctx ctx, - void* user_data); + ngtcp2_rand_ctx ctx); static int OnGetNewConnectionID( ngtcp2_conn* conn, @@ -1437,7 +1436,11 @@ class QuicSession final : public AsyncWrap, const uint8_t* token, void* user_data); - static void OnQlogWrite(void* user_data, const void* data, size_t len); + static void OnQlogWrite( + void* user_data, + uint32_t flags, + const void* data, + size_t len); #define V(id, _) QUICSESSION_FLAG_##id, enum QuicSessionFlags : uint32_t {